Some more scalafication
This commit is contained in:
parent
69d031461b
commit
75004747b3
315
pom.xml
315
pom.xml
@ -1,161 +1,162 @@
|
|||||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
<modelVersion>4.0.0</modelVersion>
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||||
<groupId>net.abhinavsarkar.ircsearch</groupId>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
<artifactId>irc-search</artifactId>
|
<groupId>net.abhinavsarkar.ircsearch</groupId>
|
||||||
<version>1.0-SNAPSHOT</version>
|
<artifactId>irc-search</artifactId>
|
||||||
<name>${project.artifactId}</name>
|
<version>1.0-SNAPSHOT</version>
|
||||||
|
<name>${project.artifactId}</name>
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
<maven.compiler.source>1.6</maven.compiler.source>
|
<maven.compiler.source>1.6</maven.compiler.source>
|
||||||
<maven.compiler.target>1.6</maven.compiler.target>
|
<maven.compiler.target>1.6</maven.compiler.target>
|
||||||
<encoding>UTF-8</encoding>
|
<encoding>UTF-8</encoding>
|
||||||
<scala.version>2.10.0</scala.version>
|
<scala.version>2.10.0</scala.version>
|
||||||
<scala.majorversion>2.10</scala.majorversion>
|
<scala.majorversion>2.10</scala.majorversion>
|
||||||
<lucene.version>4.3.0</lucene.version>
|
<lucene.version>4.3.0</lucene.version>
|
||||||
<project.dependencyDir>${project.build.directory}/dependency</project.dependencyDir>
|
<project.dependencyDir>${project.build.directory}/dependency</project.dependencyDir>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.scala-lang</groupId>
|
<groupId>org.scala-lang</groupId>
|
||||||
<artifactId>scala-library</artifactId>
|
<artifactId>scala-library</artifactId>
|
||||||
<version>${scala.version}</version>
|
<version>${scala.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.scala-lang</groupId>
|
<groupId>org.scala-lang</groupId>
|
||||||
<artifactId>scala-reflect</artifactId>
|
<artifactId>scala-reflect</artifactId>
|
||||||
<version>${scala.version}</version>
|
<version>${scala.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.scala-lang</groupId>
|
<groupId>org.scala-lang</groupId>
|
||||||
<artifactId>scala-compiler</artifactId>
|
<artifactId>scala-compiler</artifactId>
|
||||||
<version>${scala.version}</version>
|
<version>${scala.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>io.netty</groupId>
|
<groupId>io.netty</groupId>
|
||||||
<artifactId>netty</artifactId>
|
<artifactId>netty</artifactId>
|
||||||
<version>4.0.0.Alpha5</version>
|
<version>4.0.0.Alpha5</version>
|
||||||
<scope>compile</scope>
|
<scope>compile</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.typesafe</groupId>
|
<groupId>com.typesafe</groupId>
|
||||||
<artifactId>scalalogging-slf4j_${scala.majorversion}</artifactId>
|
<artifactId>scalalogging-slf4j_${scala.majorversion}</artifactId>
|
||||||
<version>1.0.1</version>
|
<version>1.0.1</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>ch.qos.logback</groupId>
|
<groupId>ch.qos.logback</groupId>
|
||||||
<artifactId>logback-classic</artifactId>
|
<artifactId>logback-classic</artifactId>
|
||||||
<version>1.0.0</version>
|
<version>1.0.0</version>
|
||||||
<scope>runtime</scope>
|
<scope>runtime</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>net.liftweb</groupId>
|
<groupId>net.liftweb</groupId>
|
||||||
<artifactId>lift-json_${scala.majorversion}</artifactId>
|
<artifactId>lift-json_${scala.majorversion}</artifactId>
|
||||||
<version>2.5-RC5</version>
|
<version>2.5-RC5</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.scala-lang</groupId>
|
<groupId>org.scala-lang</groupId>
|
||||||
<artifactId>jline</artifactId>
|
<artifactId>jline</artifactId>
|
||||||
<version>2.11.0-M2</version>
|
<version>2.11.0-M2</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.lucene</groupId>
|
<groupId>org.apache.lucene</groupId>
|
||||||
<artifactId>lucene-core</artifactId>
|
<artifactId>lucene-core</artifactId>
|
||||||
<version>${lucene.version}</version>
|
<version>${lucene.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.lucene</groupId>
|
<groupId>org.apache.lucene</groupId>
|
||||||
<artifactId>lucene-analyzers-common</artifactId>
|
<artifactId>lucene-analyzers-common</artifactId>
|
||||||
<version>${lucene.version}</version>
|
<version>${lucene.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.lucene</groupId>
|
<groupId>org.apache.lucene</groupId>
|
||||||
<artifactId>lucene-queryparser</artifactId>
|
<artifactId>lucene-queryparser</artifactId>
|
||||||
<version>${lucene.version}</version>
|
<version>${lucene.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>net.sf.opencsv</groupId>
|
<groupId>net.sf.opencsv</groupId>
|
||||||
<artifactId>opencsv</artifactId>
|
<artifactId>opencsv</artifactId>
|
||||||
<version>2.3</version>
|
<version>2.3</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.google.guava</groupId>
|
<groupId>com.google.guava</groupId>
|
||||||
<artifactId>guava</artifactId>
|
<artifactId>guava</artifactId>
|
||||||
<version>14.0.1</version>
|
<version>14.0.1</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.streum</groupId>
|
<groupId>org.streum</groupId>
|
||||||
<artifactId>configrity-core_${scala.majorversion}</artifactId>
|
<artifactId>configrity-core_${scala.majorversion}</artifactId>
|
||||||
<version>1.0.0</version>
|
<version>1.0.0</version>
|
||||||
<exclusions>
|
<exclusions>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>org.scalatest</groupId>
|
<groupId>org.scalatest</groupId>
|
||||||
<artifactId>scalatest_${scala.majorversion}</artifactId>
|
<artifactId>scalatest_${scala.majorversion}</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
</exclusions>
|
</exclusions>
|
||||||
</dependency>
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
<sourceDirectory>src/main/scala</sourceDirectory>
|
<sourceDirectory>src/main/scala</sourceDirectory>
|
||||||
<testSourceDirectory>src/test/scala</testSourceDirectory>
|
<testSourceDirectory>src/test/scala</testSourceDirectory>
|
||||||
<plugins>
|
<plugins>
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>org.apache.maven.plugins</groupId>
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
<artifactId>maven-dependency-plugin</artifactId>
|
<artifactId>maven-dependency-plugin</artifactId>
|
||||||
<version>2.3</version>
|
<version>2.3</version>
|
||||||
<executions>
|
<executions>
|
||||||
<execution>
|
<execution>
|
||||||
<id>copy-dependencies</id>
|
<id>copy-dependencies</id>
|
||||||
<phase>package</phase>
|
<phase>package</phase>
|
||||||
<goals>
|
<goals>
|
||||||
<goal>copy-dependencies</goal>
|
<goal>copy-dependencies</goal>
|
||||||
</goals>
|
</goals>
|
||||||
<configuration>
|
<configuration>
|
||||||
<outputDirectory>${project.dependencyDir}</outputDirectory>
|
<outputDirectory>${project.dependencyDir}</outputDirectory>
|
||||||
<includeScope>runtime</includeScope>
|
<includeScope>runtime</includeScope>
|
||||||
<excludeScope>provided</excludeScope>
|
<excludeScope>provided</excludeScope>
|
||||||
<excludeTypes>pom</excludeTypes>
|
<excludeTypes>pom</excludeTypes>
|
||||||
</configuration>
|
</configuration>
|
||||||
</execution>
|
</execution>
|
||||||
</executions>
|
</executions>
|
||||||
</plugin>
|
</plugin>
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>org.scala-tools</groupId>
|
<groupId>org.scala-tools</groupId>
|
||||||
<artifactId>maven-scala-plugin</artifactId>
|
<artifactId>maven-scala-plugin</artifactId>
|
||||||
<version>2.15.0</version>
|
<version>2.15.0</version>
|
||||||
<executions>
|
<executions>
|
||||||
<execution>
|
<execution>
|
||||||
<goals>
|
<goals>
|
||||||
<goal>compile</goal>
|
<goal>compile</goal>
|
||||||
<goal>testCompile</goal>
|
<goal>testCompile</goal>
|
||||||
</goals>
|
</goals>
|
||||||
<configuration>
|
<configuration>
|
||||||
<args>
|
<args>
|
||||||
<arg>-make:transitive</arg>
|
<arg>-make:transitive</arg>
|
||||||
<arg>-dependencyfile</arg>
|
<arg>-dependencyfile</arg>
|
||||||
<arg>${project.build.directory}/.scala_dependencies</arg>
|
<arg>${project.build.directory}/.scala_dependencies</arg>
|
||||||
</args>
|
</args>
|
||||||
</configuration>
|
</configuration>
|
||||||
</execution>
|
</execution>
|
||||||
</executions>
|
</executions>
|
||||||
</plugin>
|
</plugin>
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>org.apache.maven.plugins</groupId>
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
<artifactId>maven-surefire-plugin</artifactId>
|
<artifactId>maven-surefire-plugin</artifactId>
|
||||||
<version>2.6</version>
|
<version>2.6</version>
|
||||||
<configuration>
|
<configuration>
|
||||||
<useFile>false</useFile>
|
<useFile>false</useFile>
|
||||||
<disableXmlReport>true</disableXmlReport>
|
<disableXmlReport>true</disableXmlReport>
|
||||||
<!-- If you have classpath issue like NoDefClassError,... -->
|
<!-- If you have classpath issue like NoDefClassError,... -->
|
||||||
<!-- useManifestOnlyJar>false</useManifestOnlyJar -->
|
<!-- useManifestOnlyJar>false</useManifestOnlyJar -->
|
||||||
<includes>
|
<includes>
|
||||||
<include>**/*Test.*</include>
|
<include>**/*Test.*</include>
|
||||||
<include>**/*Suite.*</include>
|
<include>**/*Suite.*</include>
|
||||||
</includes>
|
</includes>
|
||||||
</configuration>
|
</configuration>
|
||||||
</plugin>
|
</plugin>
|
||||||
</plugins>
|
</plugins>
|
||||||
</build>
|
</build>
|
||||||
</project>
|
</project>
|
||||||
|
@ -91,12 +91,12 @@ private object UnifiedHandler extends ChannelInboundByteHandlerAdapter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
override def inboundBufferUpdated(ctx : ChannelHandlerContext, in: ByteBuf) {
|
override def inboundBufferUpdated(ctx : ChannelHandlerContext, in: ByteBuf) {
|
||||||
if (in.readableBytes() < 5) {
|
if (in.readableBytes < 5) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
val magic1 = in.getUnsignedByte(in.readerIndex())
|
val magic1 = in.getUnsignedByte(in.readerIndex)
|
||||||
val magic2 = in.getUnsignedByte(in.readerIndex() + 1)
|
val magic2 = in.getUnsignedByte(in.readerIndex + 1)
|
||||||
if (isHttp(magic1, magic2)) {
|
if (isHttp(magic1, magic2)) {
|
||||||
ctx.pipeline
|
ctx.pipeline
|
||||||
.addLast("decoder", new HttpRequestDecoder)
|
.addLast("decoder", new HttpRequestDecoder)
|
||||||
@ -107,7 +107,7 @@ private object UnifiedHandler extends ChannelInboundByteHandlerAdapter {
|
|||||||
.remove(this)
|
.remove(this)
|
||||||
} else {
|
} else {
|
||||||
ctx.pipeline
|
ctx.pipeline
|
||||||
.addLast("framedecoder", new DelimiterBasedFrameDecoder(1048576, Delimiters.lineDelimiter() : _*))
|
.addLast("framedecoder", new DelimiterBasedFrameDecoder(1048576, Delimiters.lineDelimiter : _*))
|
||||||
.addLast("decoder", new StringDecoder(Charset.forName("UTF-8")))
|
.addLast("decoder", new StringDecoder(Charset.forName("UTF-8")))
|
||||||
.addLast("csvhandler", new TcpIndexHandler)
|
.addLast("csvhandler", new TcpIndexHandler)
|
||||||
.remove(this)
|
.remove(this)
|
||||||
@ -147,7 +147,7 @@ private class TcpIndexHandler extends ChannelInboundMessageHandlerAdapter[String
|
|||||||
inited = true
|
inited = true
|
||||||
} else {
|
} else {
|
||||||
Indexer.index(IndexRequest(server, channel, botName,
|
Indexer.index(IndexRequest(server, channel, botName,
|
||||||
List(ChatLine(values(0), values(1).toLong, values(2)))))
|
Seq(ChatLine(values(0), values(1).toLong, values(2)))))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -155,7 +155,7 @@ private class TcpIndexHandler extends ChannelInboundMessageHandlerAdapter[String
|
|||||||
@Sharable
|
@Sharable
|
||||||
private object EchoHandler extends HttpRequestHandler {
|
private object EchoHandler extends HttpRequestHandler {
|
||||||
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
|
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
|
||||||
val content = request.getContent().toString(Charset.forName("UTF-8"))
|
val content = request.getContent.toString(Charset.forName("UTF-8"))
|
||||||
logRequest(ctx, request, sendSuccess(ctx, request, content))
|
logRequest(ctx, request, sendSuccess(ctx, request, content))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -165,7 +165,7 @@ private class IndexHandler extends HttpRequestHandler {
|
|||||||
implicit val formats = DefaultFormats
|
implicit val formats = DefaultFormats
|
||||||
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
|
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
|
||||||
future {
|
future {
|
||||||
val content = request.getContent().toString(Charset.forName("UTF-8"))
|
val content = request.getContent.toString(Charset.forName("UTF-8"))
|
||||||
val indexRequest = Serialization.read[IndexRequest](content)
|
val indexRequest = Serialization.read[IndexRequest](content)
|
||||||
Indexer.index(indexRequest)
|
Indexer.index(indexRequest)
|
||||||
}
|
}
|
||||||
@ -178,16 +178,16 @@ private object SearchHandler extends HttpRequestHandler {
|
|||||||
implicit val formats = DefaultFormats
|
implicit val formats = DefaultFormats
|
||||||
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
|
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
|
||||||
val f = future {
|
val f = future {
|
||||||
val method = request.getMethod()
|
val method = request.getMethod
|
||||||
val searchRequest = if (HttpMethod.POST.equals(method)) {
|
val searchRequest = if (HttpMethod.POST.equals(method)) {
|
||||||
val content = request.getContent().toString(Charset.forName("UTF-8"))
|
val content = request.getContent.toString(Charset.forName("UTF-8"))
|
||||||
Serialization.read[SearchRequest](content)
|
Serialization.read[SearchRequest](content)
|
||||||
} else if (HttpMethod.GET.equals(method)) {
|
} else if (HttpMethod.GET.equals(method)) {
|
||||||
val params = new QueryStringDecoder(request.getUri).getParameters.toMap
|
val params = new QueryStringDecoder(request.getUri).getParameters.toMap
|
||||||
val List(server, channel, botName, query) =
|
val Seq(server, channel, botName, query) =
|
||||||
List("server", "channel", "botName", "query").map(params(_).get(0))
|
Seq("server", "channel", "botName", "query").map(params(_).get(0))
|
||||||
val List(page, pageSize, details) =
|
val Seq(page, pageSize, details) =
|
||||||
List("page", "pageSize", "details").map(params.get(_).map({ case l => l.get(0) }))
|
Seq("page", "pageSize", "details").map(params.get(_).map({ case l => l.get(0) }))
|
||||||
|
|
||||||
var sr = SearchRequest(server, channel, botName, query)
|
var sr = SearchRequest(server, channel, botName, query)
|
||||||
if (page.isDefined)
|
if (page.isDefined)
|
||||||
|
@ -49,11 +49,11 @@ object Indexer extends Logging {
|
|||||||
private val config = Configuration.loadResource("/irc-search.conf").detach("indexing")
|
private val config = Configuration.loadResource("/irc-search.conf").detach("indexing")
|
||||||
|
|
||||||
val LuceneVersion = Version.LUCENE_43
|
val LuceneVersion = Version.LUCENE_43
|
||||||
private val ContextSize = config[Int]("context.size")
|
private val ContextSize = config[Int]("context.size")
|
||||||
private val ContextDurationSecs = config[Int]("context.durationSecs")
|
private val ContextDurationSecs = config[Int]("context.durationSecs")
|
||||||
private val RunIntervalSecs = config[Int]("runIntervalSecs")
|
private val RunIntervalSecs = config[Int]("runIntervalSecs")
|
||||||
private val FlushIntervalSecs = config[Int]("flushIntervalSecs")
|
private val FlushIntervalSecs = config[Int]("flushIntervalSecs")
|
||||||
private val RateLimitPerSec = config[Int]("rateLimitPerSec")
|
private val RateLimitPerSec = config[Int]("rateLimitPerSec")
|
||||||
|
|
||||||
private val indexQueue = new PriorityBlockingQueue[IndexRecord]
|
private val indexQueue = new PriorityBlockingQueue[IndexRecord]
|
||||||
private val scheduler = Executors.newScheduledThreadPool(2)
|
private val scheduler = Executors.newScheduledThreadPool(2)
|
||||||
@ -78,7 +78,7 @@ object Indexer extends Logging {
|
|||||||
val defAnalyzer = new StandardAnalyzer(LuceneVersion)
|
val defAnalyzer = new StandardAnalyzer(LuceneVersion)
|
||||||
val fieldAnalyzers = Map(
|
val fieldAnalyzers = Map(
|
||||||
ChatLine.USER -> new KeywordAnalyzer,
|
ChatLine.USER -> new KeywordAnalyzer,
|
||||||
ChatLine.MSG -> new EnglishAnalyzer(LuceneVersion),
|
ChatLine.MSG -> new EnglishAnalyzer(LuceneVersion),
|
||||||
ChatLine.CTXB -> new EnglishAnalyzer(LuceneVersion),
|
ChatLine.CTXB -> new EnglishAnalyzer(LuceneVersion),
|
||||||
ChatLine.CTXA -> new EnglishAnalyzer(LuceneVersion))
|
ChatLine.CTXA -> new EnglishAnalyzer(LuceneVersion))
|
||||||
|
|
||||||
@ -136,10 +136,10 @@ object Indexer extends Logging {
|
|||||||
rec.chatLine.copy(
|
rec.chatLine.copy(
|
||||||
contextBefore = recs.slice(idx - ContextSize, idx).map(_.chatLine)
|
contextBefore = recs.slice(idx - ContextSize, idx).map(_.chatLine)
|
||||||
.filter(_.timestamp >= rec.chatLine.timestamp - ContextDurationSecs * 1000)
|
.filter(_.timestamp >= rec.chatLine.timestamp - ContextDurationSecs * 1000)
|
||||||
.toList,
|
.toSeq,
|
||||||
contextAfter = recs.slice(idx + 1, 2 * ContextSize + 1).map(_.chatLine)
|
contextAfter = recs.slice(idx + 1, 2 * ContextSize + 1).map(_.chatLine)
|
||||||
.filter(_.timestamp <= rec.chatLine.timestamp + ContextDurationSecs * 1000)
|
.filter(_.timestamp <= rec.chatLine.timestamp + ContextDurationSecs * 1000)
|
||||||
.toList))
|
.toSeq))
|
||||||
}
|
}
|
||||||
|
|
||||||
def start {
|
def start {
|
||||||
@ -148,7 +148,7 @@ object Indexer extends Logging {
|
|||||||
if (!indexQueue.isEmpty) {
|
if (!indexQueue.isEmpty) {
|
||||||
val indexRecs = new ArrayList[IndexRecord]
|
val indexRecs = new ArrayList[IndexRecord]
|
||||||
indexQueue drainTo indexRecs
|
indexQueue drainTo indexRecs
|
||||||
val indexRecsMap = indexRecs groupBy { r => (r.server, r.channel, r.botName) }
|
val indexRecsMap = indexRecs.toIndexedSeq groupBy { r => (r.server, r.channel, r.botName) }
|
||||||
|
|
||||||
val windowSize = 2 * ContextSize + 1
|
val windowSize = 2 * ContextSize + 1
|
||||||
for (indexRecBatch <- indexRecsMap.values) {
|
for (indexRecBatch <- indexRecsMap.values) {
|
||||||
@ -192,7 +192,7 @@ object Indexer extends Logging {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def ctxToStr(ctx : List[ChatLine]) =
|
private def ctxToStr(ctx : Seq[ChatLine]) =
|
||||||
ctx.map { line => s"${line.timestamp} ${line.user}: ${line.message}" } mkString "\n"
|
ctx.map { line => s"${line.timestamp} ${line.user}: ${line.message}" } mkString "\n"
|
||||||
|
|
||||||
private def doIndex(indexRecord: IndexRecord) {
|
private def doIndex(indexRecord: IndexRecord) {
|
||||||
@ -205,7 +205,7 @@ object Indexer extends Logging {
|
|||||||
val msg = new TextField(ChatLine.MSG, chatLine.message, Field.Store.YES)
|
val msg = new TextField(ChatLine.MSG, chatLine.message, Field.Store.YES)
|
||||||
val ctxBfr = new TextField(ChatLine.CTXB, ctxToStr(chatLine.contextBefore), Field.Store.YES)
|
val ctxBfr = new TextField(ChatLine.CTXB, ctxToStr(chatLine.contextBefore), Field.Store.YES)
|
||||||
val ctxAft = new TextField(ChatLine.CTXA, ctxToStr(chatLine.contextAfter), Field.Store.YES)
|
val ctxAft = new TextField(ChatLine.CTXA, ctxToStr(chatLine.contextAfter), Field.Store.YES)
|
||||||
indexWriter.addDocument(List(ts, user, msg, ctxBfr, ctxAft), indexWriter.getAnalyzer)
|
indexWriter.addDocument(Seq(ts, user, msg, ctxBfr, ctxAft), indexWriter.getAnalyzer)
|
||||||
logger.debug("Indexed : [{} {} {}] [{}] {}: {}",
|
logger.debug("Indexed : [{} {} {}] [{}] {}: {}",
|
||||||
server, channel, botName, new Date(chatLine.timestamp), chatLine.user, chatLine.message)
|
server, channel, botName, new Date(chatLine.timestamp), chatLine.user, chatLine.message)
|
||||||
}
|
}
|
||||||
|
@ -51,7 +51,7 @@ object Searcher extends Logging {
|
|||||||
|
|
||||||
private def mkQueryParser(analyzer : Analyzer) =
|
private def mkQueryParser(analyzer : Analyzer) =
|
||||||
new MultiFieldQueryParser(Indexer.LuceneVersion,
|
new MultiFieldQueryParser(Indexer.LuceneVersion,
|
||||||
List(ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA).toArray, analyzer,
|
Array(ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA), analyzer,
|
||||||
Map(ChatLine.MSG -> MessageFieldBoost))
|
Map(ChatLine.MSG -> MessageFieldBoost))
|
||||||
|
|
||||||
private def filterifyQuery(query : Query) : Query =
|
private def filterifyQuery(query : Query) : Query =
|
||||||
@ -121,13 +121,13 @@ object Searcher extends Logging {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private val DocFields = List(ChatLine.USER, ChatLine.TS, ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA)
|
private val DocFields = Seq(ChatLine.USER, ChatLine.TS, ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA)
|
||||||
|
|
||||||
private def doSearch(indexDir : String, query : Query, page : Int, pageSize : Int)
|
private def doSearch(indexDir : String, query : Query, page : Int, pageSize : Int)
|
||||||
: (Int, List[(ChatLine, Float)]) = {
|
: (Int, Seq[(ChatLine, Float)]) = {
|
||||||
val searcherMgr = getSearcherMgr(indexDir)
|
val searcherMgr = getSearcherMgr(indexDir)
|
||||||
searcherMgr.maybeRefresh
|
searcherMgr.maybeRefresh
|
||||||
val indexSearcher = searcherMgr.acquire()
|
val indexSearcher = searcherMgr.acquire
|
||||||
try {
|
try {
|
||||||
val topDocs = indexSearcher.search(query, MaxHits.min((page + 1) * pageSize),
|
val topDocs = indexSearcher.search(query, MaxHits.min((page + 1) * pageSize),
|
||||||
new Sort(SortField.FIELD_SCORE, new SortField(ChatLine.TS, SortField.Type.LONG, true)))
|
new Sort(SortField.FIELD_SCORE, new SortField(ChatLine.TS, SortField.Type.LONG, true)))
|
||||||
@ -139,15 +139,15 @@ object Searcher extends Logging {
|
|||||||
(map, field) => map += (field.name -> field.stringValue)
|
(map, field) => map += (field.name -> field.stringValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
val List(user, timestamp, message, contextBefore, contextAfter) = DocFields.map(doc)
|
val Seq(user, timestamp, message, contextBefore, contextAfter) = DocFields.map(doc)
|
||||||
|
|
||||||
val LineRe = "(\\d+) (.*?): (.*)".r
|
val LineRe = "(\\d+) (.*?): (.*)".r
|
||||||
val List(ctxBefore, ctxAfter) = List(contextBefore, contextAfter).map {
|
val Seq(ctxBefore, ctxAfter) = Seq(contextBefore, contextAfter).map {
|
||||||
_.split('\n').filterNot(_.isEmpty).map {
|
_.split('\n').filterNot(_.isEmpty).map {
|
||||||
case LineRe(timestamp, user, message) => ChatLine(user, timestamp.toLong, message)
|
case LineRe(timestamp, user, message) => ChatLine(user, timestamp.toLong, message)
|
||||||
}}
|
}}
|
||||||
|
|
||||||
val chatLine = ChatLine(user, timestamp.toLong, message, ctxBefore.toList, ctxAfter.toList)
|
val chatLine = ChatLine(user, timestamp.toLong, message, ctxBefore, ctxAfter)
|
||||||
(chatLine, score)
|
(chatLine, score)
|
||||||
}
|
}
|
||||||
(topDocs.totalHits, docs.toList)
|
(topDocs.totalHits, docs.toList)
|
||||||
|
@ -10,11 +10,11 @@ object ChatLine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
case class ChatLine(user : String, timestamp : Long, message : String,
|
case class ChatLine(user : String, timestamp : Long, message : String,
|
||||||
contextBefore : List[ChatLine] = List(),
|
contextBefore : Seq[ChatLine] = Seq(),
|
||||||
contextAfter : List[ChatLine] = List())
|
contextAfter : Seq[ChatLine] = Seq())
|
||||||
|
|
||||||
case class IndexRequest(
|
case class IndexRequest(
|
||||||
server : String, channel : String, botName : String, chatLines : List[ChatLine])
|
server : String, channel : String, botName : String, chatLines : Seq[ChatLine])
|
||||||
|
|
||||||
case class SearchRequest(
|
case class SearchRequest(
|
||||||
server : String, channel : String, botName : String, query: String,
|
server : String, channel : String, botName : String, query: String,
|
||||||
@ -22,13 +22,13 @@ case class SearchRequest(
|
|||||||
|
|
||||||
case class SearchResult(
|
case class SearchResult(
|
||||||
server : String, channel : String, botName : String, query: String,
|
server : String, channel : String, botName : String, query: String,
|
||||||
page : Int, pageSize : Int, totalResults : Int, chatLines : List[ChatLine]) {
|
page : Int, pageSize : Int, totalResults : Int, chatLines : Seq[ChatLine]) {
|
||||||
def toSimpleSearchResult =
|
def toSimpleSearchResult =
|
||||||
SimpleSearchResult(server, channel, botName, query, page, pageSize, totalResults,
|
SimpleSearchResult(server, channel, botName, query, page, pageSize, totalResults,
|
||||||
chatLines map {
|
chatLines map {
|
||||||
case mline@ChatLine(_, _, _, contextBefore, contextAfter) =>
|
case mline@ChatLine(_, _, _, contextBefore, contextAfter) =>
|
||||||
((contextBefore :+ mline) ++ contextAfter) map { line =>
|
((contextBefore :+ mline) ++ contextAfter) map { line =>
|
||||||
List(line.timestamp.toString, line.user, line.message)
|
Seq(line.timestamp.toString, line.user, line.message)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -36,12 +36,12 @@ case class SearchResult(
|
|||||||
object SearchResult {
|
object SearchResult {
|
||||||
def fromSearchRequest(searchRequest : SearchRequest) = searchRequest match {
|
def fromSearchRequest(searchRequest : SearchRequest) = searchRequest match {
|
||||||
case SearchRequest(server, channel, botName, query, page, pageSize, _) =>
|
case SearchRequest(server, channel, botName, query, page, pageSize, _) =>
|
||||||
new SearchResult(server, channel, botName, query, page, pageSize, 0, List())
|
new SearchResult(server, channel, botName, query, page, pageSize, 0, Seq())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
case class SimpleSearchResult(
|
case class SimpleSearchResult(
|
||||||
server : String, channel : String, botName : String, query: String,
|
server : String, channel : String, botName : String, query: String,
|
||||||
page : Int, pageSize : Int, totalResults : Int, lines : List[List[List[String]]])
|
page : Int, pageSize : Int, totalResults : Int, lines : Seq[Seq[Seq[String]]])
|
||||||
|
|
||||||
case class SearchError(error : String)
|
case class SearchError(error : String)
|
Loading…
Reference in New Issue
Block a user