abhin4v
/
irc-search
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Some more scalafication

Browse Source
master
Abhinav Sarkar 10 years ago
parent 69d031461b
commit 75004747b3
5 changed files with 195 additions and 194 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 315
      pom.xml
  2. 26
      src/main/scala/net/abhinavsarkar/ircsearch/Server.scala
  3. 20
      src/main/scala/net/abhinavsarkar/ircsearch/lucene/Indexer.scala
  4. 14
      src/main/scala/net/abhinavsarkar/ircsearch/lucene/Searcher.scala
  5. 14
      src/main/scala/net/abhinavsarkar/ircsearch/model.scala

315
pom.xml
Unescape View File

@ -1,161 +1,162 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>net.abhinavsarkar.ircsearch</groupId>
<artifactId>irc-search</artifactId>
<version>1.0-SNAPSHOT</version>
<name>${project.artifactId}</name>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>net.abhinavsarkar.ircsearch</groupId>
<artifactId>irc-search</artifactId>
<version>1.0-SNAPSHOT</version>
<name>${project.artifactId}</name>
<properties>
<maven.compiler.source>1.6</maven.compiler.source>
<maven.compiler.target>1.6</maven.compiler.target>
<encoding>UTF-8</encoding>
<scala.version>2.10.0</scala.version>
<scala.majorversion>2.10</scala.majorversion>
<lucene.version>4.3.0</lucene.version>
<project.dependencyDir>${project.build.directory}/dependency</project.dependencyDir>
</properties>
<properties>
<maven.compiler.source>1.6</maven.compiler.source>
<maven.compiler.target>1.6</maven.compiler.target>
<encoding>UTF-8</encoding>
<scala.version>2.10.0</scala.version>
<scala.majorversion>2.10</scala.majorversion>
<lucene.version>4.3.0</lucene.version>
<project.dependencyDir>${project.build.directory}/dependency</project.dependencyDir>
</properties>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
<version>4.0.0.Alpha5</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>scalalogging-slf4j_${scala.majorversion}</artifactId>
<version>1.0.1</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.0.0</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>net.liftweb</groupId>
<artifactId>lift-json_${scala.majorversion}</artifactId>
<version>2.5-RC5</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>jline</artifactId>
<version>2.11.0-M2</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>14.0.1</version>
</dependency>
<dependency>
<groupId>org.streum</groupId>
<artifactId>configrity-core_${scala.majorversion}</artifactId>
<version>1.0.0</version>
<exclusions>
<exclusion>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.majorversion}</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
<version>4.0.0.Alpha5</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>scalalogging-slf4j_${scala.majorversion}</artifactId>
<version>1.0.1</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.0.0</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>net.liftweb</groupId>
<artifactId>lift-json_${scala.majorversion}</artifactId>
<version>2.5-RC5</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>jline</artifactId>
<version>2.11.0-M2</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>14.0.1</version>
</dependency>
<dependency>
<groupId>org.streum</groupId>
<artifactId>configrity-core_${scala.majorversion}</artifactId>
<version>1.0.0</version>
<exclusions>
<exclusion>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.majorversion}</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.3</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.dependencyDir}</outputDirectory>
<includeScope>runtime</includeScope>
<excludeScope>provided</excludeScope>
<excludeTypes>pom</excludeTypes>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<args>
<arg>-make:transitive</arg>
<arg>-dependencyfile</arg>
<arg>${project.build.directory}/.scala_dependencies</arg>
</args>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.6</version>
<configuration>
<useFile>false</useFile>
<disableXmlReport>true</disableXmlReport>
<!-- If you have classpath issue like NoDefClassError,... -->
<!-- useManifestOnlyJar>false</useManifestOnlyJar -->
<includes>
<include>**/*Test.*</include>
<include>**/*Suite.*</include>
</includes>
</configuration>
</plugin>
</plugins>
</build>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.3</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.dependencyDir}</outputDirectory>
<includeScope>runtime</includeScope>
<excludeScope>provided</excludeScope>
<excludeTypes>pom</excludeTypes>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<args>
<arg>-make:transitive</arg>
<arg>-dependencyfile</arg>
<arg>${project.build.directory}/.scala_dependencies</arg>
</args>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.6</version>
<configuration>
<useFile>false</useFile>
<disableXmlReport>true</disableXmlReport>
<!-- If you have classpath issue like NoDefClassError,... -->
<!-- useManifestOnlyJar>false</useManifestOnlyJar -->
<includes>
<include>**/*Test.*</include>
<include>**/*Suite.*</include>
</includes>
</configuration>
</plugin>
</plugins>
</build>
</project>

26
src/main/scala/net/abhinavsarkar/ircsearch/Server.scala
Unescape View File

@ -91,12 +91,12 @@ private object UnifiedHandler extends ChannelInboundByteHandlerAdapter {
}
override def inboundBufferUpdated(ctx : ChannelHandlerContext, in: ByteBuf) {
if (in.readableBytes() < 5) {
if (in.readableBytes < 5) {
return;
}
val magic1 = in.getUnsignedByte(in.readerIndex())
val magic2 = in.getUnsignedByte(in.readerIndex() + 1)
val magic1 = in.getUnsignedByte(in.readerIndex)
val magic2 = in.getUnsignedByte(in.readerIndex + 1)
if (isHttp(magic1, magic2)) {
ctx.pipeline
.addLast("decoder", new HttpRequestDecoder)
@ -107,7 +107,7 @@ private object UnifiedHandler extends ChannelInboundByteHandlerAdapter {
.remove(this)
} else {
ctx.pipeline
.addLast("framedecoder", new DelimiterBasedFrameDecoder(1048576, Delimiters.lineDelimiter() : _*))
.addLast("framedecoder", new DelimiterBasedFrameDecoder(1048576, Delimiters.lineDelimiter : _*))
.addLast("decoder", new StringDecoder(Charset.forName("UTF-8")))
.addLast("csvhandler", new TcpIndexHandler)
.remove(this)
@ -147,7 +147,7 @@ private class TcpIndexHandler extends ChannelInboundMessageHandlerAdapter[String
inited = true
} else {
Indexer.index(IndexRequest(server, channel, botName,
List(ChatLine(values(0), values(1).toLong, values(2)))))
Seq(ChatLine(values(0), values(1).toLong, values(2)))))
}
}
}
@ -155,7 +155,7 @@ private class TcpIndexHandler extends ChannelInboundMessageHandlerAdapter[String
@Sharable
private object EchoHandler extends HttpRequestHandler {
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
val content = request.getContent().toString(Charset.forName("UTF-8"))
val content = request.getContent.toString(Charset.forName("UTF-8"))
logRequest(ctx, request, sendSuccess(ctx, request, content))
}
}
@ -165,7 +165,7 @@ private class IndexHandler extends HttpRequestHandler {
implicit val formats = DefaultFormats
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
future {
val content = request.getContent().toString(Charset.forName("UTF-8"))
val content = request.getContent.toString(Charset.forName("UTF-8"))
val indexRequest = Serialization.read[IndexRequest](content)
Indexer.index(indexRequest)
}
@ -178,16 +178,16 @@ private object SearchHandler extends HttpRequestHandler {
implicit val formats = DefaultFormats
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
val f = future {
val method = request.getMethod()
val method = request.getMethod
val searchRequest = if (HttpMethod.POST.equals(method)) {
val content = request.getContent().toString(Charset.forName("UTF-8"))
val content = request.getContent.toString(Charset.forName("UTF-8"))
Serialization.read[SearchRequest](content)
} else if (HttpMethod.GET.equals(method)) {
val params = new QueryStringDecoder(request.getUri).getParameters.toMap
val List(server, channel, botName, query) =
List("server", "channel", "botName", "query").map(params(_).get(0))
val List(page, pageSize, details) =
List("page", "pageSize", "details").map(params.get(_).map({ case l => l.get(0) }))
val Seq(server, channel, botName, query) =
Seq("server", "channel", "botName", "query").map(params(_).get(0))
val Seq(page, pageSize, details) =
Seq("page", "pageSize", "details").map(params.get(_).map({ case l => l.get(0) }))
var sr = SearchRequest(server, channel, botName, query)
if (page.isDefined)

20
src/main/scala/net/abhinavsarkar/ircsearch/lucene/Indexer.scala
Unescape View File

@ -49,11 +49,11 @@ object Indexer extends Logging {
private val config = Configuration.loadResource("/irc-search.conf").detach("indexing")
val LuceneVersion = Version.LUCENE_43
private val ContextSize = config[Int]("context.size")
private val ContextSize = config[Int]("context.size")
private val ContextDurationSecs = config[Int]("context.durationSecs")
private val RunIntervalSecs = config[Int]("runIntervalSecs")
private val FlushIntervalSecs = config[Int]("flushIntervalSecs")
private val RateLimitPerSec = config[Int]("rateLimitPerSec")
private val RunIntervalSecs = config[Int]("runIntervalSecs")
private val FlushIntervalSecs = config[Int]("flushIntervalSecs")
private val RateLimitPerSec = config[Int]("rateLimitPerSec")
private val indexQueue = new PriorityBlockingQueue[IndexRecord]
private val scheduler = Executors.newScheduledThreadPool(2)
@ -78,7 +78,7 @@ object Indexer extends Logging {
val defAnalyzer = new StandardAnalyzer(LuceneVersion)
val fieldAnalyzers = Map(
ChatLine.USER -> new KeywordAnalyzer,
ChatLine.MSG -> new EnglishAnalyzer(LuceneVersion),
ChatLine.MSG -> new EnglishAnalyzer(LuceneVersion),
ChatLine.CTXB -> new EnglishAnalyzer(LuceneVersion),
ChatLine.CTXA -> new EnglishAnalyzer(LuceneVersion))
@ -136,10 +136,10 @@ object Indexer extends Logging {
rec.chatLine.copy(
contextBefore = recs.slice(idx - ContextSize, idx).map(_.chatLine)
.filter(_.timestamp >= rec.chatLine.timestamp - ContextDurationSecs * 1000)
.toList,
.toSeq,
contextAfter = recs.slice(idx + 1, 2 * ContextSize + 1).map(_.chatLine)
.filter(_.timestamp <= rec.chatLine.timestamp + ContextDurationSecs * 1000)
.toList))
.toSeq))
}
def start {
@ -148,7 +148,7 @@ object Indexer extends Logging {
if (!indexQueue.isEmpty) {
val indexRecs = new ArrayList[IndexRecord]
indexQueue drainTo indexRecs
val indexRecsMap = indexRecs groupBy { r => (r.server, r.channel, r.botName) }
val indexRecsMap = indexRecs.toIndexedSeq groupBy { r => (r.server, r.channel, r.botName) }
val windowSize = 2 * ContextSize + 1
for (indexRecBatch <- indexRecsMap.values) {
@ -192,7 +192,7 @@ object Indexer extends Logging {
}
}
private def ctxToStr(ctx : List[ChatLine]) =
private def ctxToStr(ctx : Seq[ChatLine]) =
ctx.map { line => s"${line.timestamp} ${line.user}: ${line.message}" } mkString "\n"
private def doIndex(indexRecord: IndexRecord) {
@ -205,7 +205,7 @@ object Indexer extends Logging {
val msg = new TextField(ChatLine.MSG, chatLine.message, Field.Store.YES)
val ctxBfr = new TextField(ChatLine.CTXB, ctxToStr(chatLine.contextBefore), Field.Store.YES)
val ctxAft = new TextField(ChatLine.CTXA, ctxToStr(chatLine.contextAfter), Field.Store.YES)
indexWriter.addDocument(List(ts, user, msg, ctxBfr, ctxAft), indexWriter.getAnalyzer)
indexWriter.addDocument(Seq(ts, user, msg, ctxBfr, ctxAft), indexWriter.getAnalyzer)
logger.debug("Indexed : [{} {} {}] [{}] {}: {}",
server, channel, botName, new Date(chatLine.timestamp), chatLine.user, chatLine.message)
}

14
src/main/scala/net/abhinavsarkar/ircsearch/lucene/Searcher.scala
Unescape View File

@ -51,7 +51,7 @@ object Searcher extends Logging {
private def mkQueryParser(analyzer : Analyzer) =
new MultiFieldQueryParser(Indexer.LuceneVersion,
List(ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA).toArray, analyzer,
Array(ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA), analyzer,
Map(ChatLine.MSG -> MessageFieldBoost))
private def filterifyQuery(query : Query) : Query =
@ -121,13 +121,13 @@ object Searcher extends Logging {
}
}
private val DocFields = List(ChatLine.USER, ChatLine.TS, ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA)
private val DocFields = Seq(ChatLine.USER, ChatLine.TS, ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA)
private def doSearch(indexDir : String, query : Query, page : Int, pageSize : Int)
: (Int, List[(ChatLine, Float)]) = {
: (Int, Seq[(ChatLine, Float)]) = {
val searcherMgr = getSearcherMgr(indexDir)
searcherMgr.maybeRefresh
val indexSearcher = searcherMgr.acquire()
val indexSearcher = searcherMgr.acquire
try {
val topDocs = indexSearcher.search(query, MaxHits.min((page + 1) * pageSize),
new Sort(SortField.FIELD_SCORE, new SortField(ChatLine.TS, SortField.Type.LONG, true)))
@ -139,15 +139,15 @@ object Searcher extends Logging {
(map, field) => map += (field.name -> field.stringValue)
}
val List(user, timestamp, message, contextBefore, contextAfter) = DocFields.map(doc)
val Seq(user, timestamp, message, contextBefore, contextAfter) = DocFields.map(doc)
val LineRe = "(\\d+) (.*?): (.*)".r
val List(ctxBefore, ctxAfter) = List(contextBefore, contextAfter).map {
val Seq(ctxBefore, ctxAfter) = Seq(contextBefore, contextAfter).map {
_.split('\n').filterNot(_.isEmpty).map {
case LineRe(timestamp, user, message) => ChatLine(user, timestamp.toLong, message)
}}
val chatLine = ChatLine(user, timestamp.toLong, message, ctxBefore.toList, ctxAfter.toList)
val chatLine = ChatLine(user, timestamp.toLong, message, ctxBefore, ctxAfter)
(chatLine, score)
}
(topDocs.totalHits, docs.toList)

14
src/main/scala/net/abhinavsarkar/ircsearch/model.scala
Unescape View File

@ -10,11 +10,11 @@ object ChatLine {
}
case class ChatLine(user : String, timestamp : Long, message : String,
contextBefore : List[ChatLine] = List(),
contextAfter : List[ChatLine] = List())
contextBefore : Seq[ChatLine] = Seq(),
contextAfter : Seq[ChatLine] = Seq())
case class IndexRequest(
server : String, channel : String, botName : String, chatLines : List[ChatLine])
server : String, channel : String, botName : String, chatLines : Seq[ChatLine])
case class SearchRequest(
server : String, channel : String, botName : String, query: String,
@ -22,13 +22,13 @@ case class SearchRequest(
case class SearchResult(
server : String, channel : String, botName : String, query: String,
page : Int, pageSize : Int, totalResults : Int, chatLines : List[ChatLine]) {
page : Int, pageSize : Int, totalResults : Int, chatLines : Seq[ChatLine]) {
def toSimpleSearchResult =
SimpleSearchResult(server, channel, botName, query, page, pageSize, totalResults,
chatLines map {
case mline@ChatLine(_, _, _, contextBefore, contextAfter) =>
((contextBefore :+ mline) ++ contextAfter) map { line =>
List(line.timestamp.toString, line.user, line.message)
Seq(line.timestamp.toString, line.user, line.message)
}
})
}
@ -36,12 +36,12 @@ case class SearchResult(
object SearchResult {
def fromSearchRequest(searchRequest : SearchRequest) = searchRequest match {
case SearchRequest(server, channel, botName, query, page, pageSize, _) =>
new SearchResult(server, channel, botName, query, page, pageSize, 0, List())
new SearchResult(server, channel, botName, query, page, pageSize, 0, Seq())
}
}
case class SimpleSearchResult(
server : String, channel : String, botName : String, query: String,
page : Int, pageSize : Int, totalResults : Int, lines : List[List[List[String]]])
page : Int, pageSize : Int, totalResults : Int, lines : Seq[Seq[Seq[String]]])
case class SearchError(error : String)
Write Preview
Loading…
Cancel
Save