From 75004747b33ec3639bc475bccf6329170b2be6eb Mon Sep 17 00:00:00 2001 From: Abhinav Sarkar Date: Thu, 23 May 2013 10:26:34 +0530 Subject: [PATCH] Some more scalafication --- pom.xml | 315 +++++++++--------- .../net/abhinavsarkar/ircsearch/Server.scala | 26 +- .../ircsearch/lucene/Indexer.scala | 20 +- .../ircsearch/lucene/Searcher.scala | 14 +- .../net/abhinavsarkar/ircsearch/model.scala | 14 +- 5 files changed, 195 insertions(+), 194 deletions(-) diff --git a/pom.xml b/pom.xml index bc4140b..f1c223e 100644 --- a/pom.xml +++ b/pom.xml @@ -1,161 +1,162 @@ - - 4.0.0 - net.abhinavsarkar.ircsearch - irc-search - 1.0-SNAPSHOT - ${project.artifactId} + + 4.0.0 + net.abhinavsarkar.ircsearch + irc-search + 1.0-SNAPSHOT + ${project.artifactId} - - 1.6 - 1.6 - UTF-8 - 2.10.0 - 2.10 - 4.3.0 - ${project.build.directory}/dependency - + + 1.6 + 1.6 + UTF-8 + 2.10.0 + 2.10 + 4.3.0 + ${project.build.directory}/dependency + - - - org.scala-lang - scala-library - ${scala.version} - - - org.scala-lang - scala-reflect - ${scala.version} - - - org.scala-lang - scala-compiler - ${scala.version} - - - io.netty - netty - 4.0.0.Alpha5 - compile - - - com.typesafe - scalalogging-slf4j_${scala.majorversion} - 1.0.1 - - - ch.qos.logback - logback-classic - 1.0.0 - runtime - - - net.liftweb - lift-json_${scala.majorversion} - 2.5-RC5 - - - org.scala-lang - jline - 2.11.0-M2 - - - org.apache.lucene - lucene-core - ${lucene.version} - - - org.apache.lucene - lucene-analyzers-common - ${lucene.version} - - - org.apache.lucene - lucene-queryparser - ${lucene.version} - - - net.sf.opencsv - opencsv - 2.3 - - - com.google.guava - guava - 14.0.1 - - - org.streum - configrity-core_${scala.majorversion} - 1.0.0 - - - org.scalatest - scalatest_${scala.majorversion} - - - - + + + org.scala-lang + scala-library + ${scala.version} + + + org.scala-lang + scala-reflect + ${scala.version} + + + org.scala-lang + scala-compiler + ${scala.version} + + + io.netty + netty + 4.0.0.Alpha5 + compile + + + com.typesafe + scalalogging-slf4j_${scala.majorversion} + 1.0.1 + + + ch.qos.logback + logback-classic + 1.0.0 + runtime + + + net.liftweb + lift-json_${scala.majorversion} + 2.5-RC5 + + + org.scala-lang + jline + 2.11.0-M2 + + + org.apache.lucene + lucene-core + ${lucene.version} + + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} + + + org.apache.lucene + lucene-queryparser + ${lucene.version} + + + net.sf.opencsv + opencsv + 2.3 + + + com.google.guava + guava + 14.0.1 + + + org.streum + configrity-core_${scala.majorversion} + 1.0.0 + + + org.scalatest + scalatest_${scala.majorversion} + + + + - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-dependency-plugin - 2.3 - - - copy-dependencies - package - - copy-dependencies - - - ${project.dependencyDir} - runtime - provided - pom - - - - - - org.scala-tools - maven-scala-plugin - 2.15.0 - - - - compile - testCompile - - - - -make:transitive - -dependencyfile - ${project.build.directory}/.scala_dependencies - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.6 - - false - true - - - - **/*Test.* - **/*Suite.* - - - - - + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-dependency-plugin + 2.3 + + + copy-dependencies + package + + copy-dependencies + + + ${project.dependencyDir} + runtime + provided + pom + + + + + + org.scala-tools + maven-scala-plugin + 2.15.0 + + + + compile + testCompile + + + + -make:transitive + -dependencyfile + ${project.build.directory}/.scala_dependencies + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.6 + + false + true + + + + **/*Test.* + **/*Suite.* + + + + + diff --git a/src/main/scala/net/abhinavsarkar/ircsearch/Server.scala b/src/main/scala/net/abhinavsarkar/ircsearch/Server.scala index 909e857..4f25c48 100644 --- a/src/main/scala/net/abhinavsarkar/ircsearch/Server.scala +++ b/src/main/scala/net/abhinavsarkar/ircsearch/Server.scala @@ -91,12 +91,12 @@ private object UnifiedHandler extends ChannelInboundByteHandlerAdapter { } override def inboundBufferUpdated(ctx : ChannelHandlerContext, in: ByteBuf) { - if (in.readableBytes() < 5) { + if (in.readableBytes < 5) { return; } - val magic1 = in.getUnsignedByte(in.readerIndex()) - val magic2 = in.getUnsignedByte(in.readerIndex() + 1) + val magic1 = in.getUnsignedByte(in.readerIndex) + val magic2 = in.getUnsignedByte(in.readerIndex + 1) if (isHttp(magic1, magic2)) { ctx.pipeline .addLast("decoder", new HttpRequestDecoder) @@ -107,7 +107,7 @@ private object UnifiedHandler extends ChannelInboundByteHandlerAdapter { .remove(this) } else { ctx.pipeline - .addLast("framedecoder", new DelimiterBasedFrameDecoder(1048576, Delimiters.lineDelimiter() : _*)) + .addLast("framedecoder", new DelimiterBasedFrameDecoder(1048576, Delimiters.lineDelimiter : _*)) .addLast("decoder", new StringDecoder(Charset.forName("UTF-8"))) .addLast("csvhandler", new TcpIndexHandler) .remove(this) @@ -147,7 +147,7 @@ private class TcpIndexHandler extends ChannelInboundMessageHandlerAdapter[String inited = true } else { Indexer.index(IndexRequest(server, channel, botName, - List(ChatLine(values(0), values(1).toLong, values(2))))) + Seq(ChatLine(values(0), values(1).toLong, values(2))))) } } } @@ -155,7 +155,7 @@ private class TcpIndexHandler extends ChannelInboundMessageHandlerAdapter[String @Sharable private object EchoHandler extends HttpRequestHandler { override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) { - val content = request.getContent().toString(Charset.forName("UTF-8")) + val content = request.getContent.toString(Charset.forName("UTF-8")) logRequest(ctx, request, sendSuccess(ctx, request, content)) } } @@ -165,7 +165,7 @@ private class IndexHandler extends HttpRequestHandler { implicit val formats = DefaultFormats override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) { future { - val content = request.getContent().toString(Charset.forName("UTF-8")) + val content = request.getContent.toString(Charset.forName("UTF-8")) val indexRequest = Serialization.read[IndexRequest](content) Indexer.index(indexRequest) } @@ -178,16 +178,16 @@ private object SearchHandler extends HttpRequestHandler { implicit val formats = DefaultFormats override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) { val f = future { - val method = request.getMethod() + val method = request.getMethod val searchRequest = if (HttpMethod.POST.equals(method)) { - val content = request.getContent().toString(Charset.forName("UTF-8")) + val content = request.getContent.toString(Charset.forName("UTF-8")) Serialization.read[SearchRequest](content) } else if (HttpMethod.GET.equals(method)) { val params = new QueryStringDecoder(request.getUri).getParameters.toMap - val List(server, channel, botName, query) = - List("server", "channel", "botName", "query").map(params(_).get(0)) - val List(page, pageSize, details) = - List("page", "pageSize", "details").map(params.get(_).map({ case l => l.get(0) })) + val Seq(server, channel, botName, query) = + Seq("server", "channel", "botName", "query").map(params(_).get(0)) + val Seq(page, pageSize, details) = + Seq("page", "pageSize", "details").map(params.get(_).map({ case l => l.get(0) })) var sr = SearchRequest(server, channel, botName, query) if (page.isDefined) diff --git a/src/main/scala/net/abhinavsarkar/ircsearch/lucene/Indexer.scala b/src/main/scala/net/abhinavsarkar/ircsearch/lucene/Indexer.scala index c1c3f09..e2f8c52 100644 --- a/src/main/scala/net/abhinavsarkar/ircsearch/lucene/Indexer.scala +++ b/src/main/scala/net/abhinavsarkar/ircsearch/lucene/Indexer.scala @@ -49,11 +49,11 @@ object Indexer extends Logging { private val config = Configuration.loadResource("/irc-search.conf").detach("indexing") val LuceneVersion = Version.LUCENE_43 - private val ContextSize = config[Int]("context.size") + private val ContextSize = config[Int]("context.size") private val ContextDurationSecs = config[Int]("context.durationSecs") - private val RunIntervalSecs = config[Int]("runIntervalSecs") - private val FlushIntervalSecs = config[Int]("flushIntervalSecs") - private val RateLimitPerSec = config[Int]("rateLimitPerSec") + private val RunIntervalSecs = config[Int]("runIntervalSecs") + private val FlushIntervalSecs = config[Int]("flushIntervalSecs") + private val RateLimitPerSec = config[Int]("rateLimitPerSec") private val indexQueue = new PriorityBlockingQueue[IndexRecord] private val scheduler = Executors.newScheduledThreadPool(2) @@ -78,7 +78,7 @@ object Indexer extends Logging { val defAnalyzer = new StandardAnalyzer(LuceneVersion) val fieldAnalyzers = Map( ChatLine.USER -> new KeywordAnalyzer, - ChatLine.MSG -> new EnglishAnalyzer(LuceneVersion), + ChatLine.MSG -> new EnglishAnalyzer(LuceneVersion), ChatLine.CTXB -> new EnglishAnalyzer(LuceneVersion), ChatLine.CTXA -> new EnglishAnalyzer(LuceneVersion)) @@ -136,10 +136,10 @@ object Indexer extends Logging { rec.chatLine.copy( contextBefore = recs.slice(idx - ContextSize, idx).map(_.chatLine) .filter(_.timestamp >= rec.chatLine.timestamp - ContextDurationSecs * 1000) - .toList, + .toSeq, contextAfter = recs.slice(idx + 1, 2 * ContextSize + 1).map(_.chatLine) .filter(_.timestamp <= rec.chatLine.timestamp + ContextDurationSecs * 1000) - .toList)) + .toSeq)) } def start { @@ -148,7 +148,7 @@ object Indexer extends Logging { if (!indexQueue.isEmpty) { val indexRecs = new ArrayList[IndexRecord] indexQueue drainTo indexRecs - val indexRecsMap = indexRecs groupBy { r => (r.server, r.channel, r.botName) } + val indexRecsMap = indexRecs.toIndexedSeq groupBy { r => (r.server, r.channel, r.botName) } val windowSize = 2 * ContextSize + 1 for (indexRecBatch <- indexRecsMap.values) { @@ -192,7 +192,7 @@ object Indexer extends Logging { } } - private def ctxToStr(ctx : List[ChatLine]) = + private def ctxToStr(ctx : Seq[ChatLine]) = ctx.map { line => s"${line.timestamp} ${line.user}: ${line.message}" } mkString "\n" private def doIndex(indexRecord: IndexRecord) { @@ -205,7 +205,7 @@ object Indexer extends Logging { val msg = new TextField(ChatLine.MSG, chatLine.message, Field.Store.YES) val ctxBfr = new TextField(ChatLine.CTXB, ctxToStr(chatLine.contextBefore), Field.Store.YES) val ctxAft = new TextField(ChatLine.CTXA, ctxToStr(chatLine.contextAfter), Field.Store.YES) - indexWriter.addDocument(List(ts, user, msg, ctxBfr, ctxAft), indexWriter.getAnalyzer) + indexWriter.addDocument(Seq(ts, user, msg, ctxBfr, ctxAft), indexWriter.getAnalyzer) logger.debug("Indexed : [{} {} {}] [{}] {}: {}", server, channel, botName, new Date(chatLine.timestamp), chatLine.user, chatLine.message) } diff --git a/src/main/scala/net/abhinavsarkar/ircsearch/lucene/Searcher.scala b/src/main/scala/net/abhinavsarkar/ircsearch/lucene/Searcher.scala index c2f91e6..c3d3c0d 100644 --- a/src/main/scala/net/abhinavsarkar/ircsearch/lucene/Searcher.scala +++ b/src/main/scala/net/abhinavsarkar/ircsearch/lucene/Searcher.scala @@ -51,7 +51,7 @@ object Searcher extends Logging { private def mkQueryParser(analyzer : Analyzer) = new MultiFieldQueryParser(Indexer.LuceneVersion, - List(ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA).toArray, analyzer, + Array(ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA), analyzer, Map(ChatLine.MSG -> MessageFieldBoost)) private def filterifyQuery(query : Query) : Query = @@ -121,13 +121,13 @@ object Searcher extends Logging { } } - private val DocFields = List(ChatLine.USER, ChatLine.TS, ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA) + private val DocFields = Seq(ChatLine.USER, ChatLine.TS, ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA) private def doSearch(indexDir : String, query : Query, page : Int, pageSize : Int) - : (Int, List[(ChatLine, Float)]) = { + : (Int, Seq[(ChatLine, Float)]) = { val searcherMgr = getSearcherMgr(indexDir) searcherMgr.maybeRefresh - val indexSearcher = searcherMgr.acquire() + val indexSearcher = searcherMgr.acquire try { val topDocs = indexSearcher.search(query, MaxHits.min((page + 1) * pageSize), new Sort(SortField.FIELD_SCORE, new SortField(ChatLine.TS, SortField.Type.LONG, true))) @@ -139,15 +139,15 @@ object Searcher extends Logging { (map, field) => map += (field.name -> field.stringValue) } - val List(user, timestamp, message, contextBefore, contextAfter) = DocFields.map(doc) + val Seq(user, timestamp, message, contextBefore, contextAfter) = DocFields.map(doc) val LineRe = "(\\d+) (.*?): (.*)".r - val List(ctxBefore, ctxAfter) = List(contextBefore, contextAfter).map { + val Seq(ctxBefore, ctxAfter) = Seq(contextBefore, contextAfter).map { _.split('\n').filterNot(_.isEmpty).map { case LineRe(timestamp, user, message) => ChatLine(user, timestamp.toLong, message) }} - val chatLine = ChatLine(user, timestamp.toLong, message, ctxBefore.toList, ctxAfter.toList) + val chatLine = ChatLine(user, timestamp.toLong, message, ctxBefore, ctxAfter) (chatLine, score) } (topDocs.totalHits, docs.toList) diff --git a/src/main/scala/net/abhinavsarkar/ircsearch/model.scala b/src/main/scala/net/abhinavsarkar/ircsearch/model.scala index a24a61b..e77ea90 100644 --- a/src/main/scala/net/abhinavsarkar/ircsearch/model.scala +++ b/src/main/scala/net/abhinavsarkar/ircsearch/model.scala @@ -10,11 +10,11 @@ object ChatLine { } case class ChatLine(user : String, timestamp : Long, message : String, - contextBefore : List[ChatLine] = List(), - contextAfter : List[ChatLine] = List()) + contextBefore : Seq[ChatLine] = Seq(), + contextAfter : Seq[ChatLine] = Seq()) case class IndexRequest( - server : String, channel : String, botName : String, chatLines : List[ChatLine]) + server : String, channel : String, botName : String, chatLines : Seq[ChatLine]) case class SearchRequest( server : String, channel : String, botName : String, query: String, @@ -22,13 +22,13 @@ case class SearchRequest( case class SearchResult( server : String, channel : String, botName : String, query: String, - page : Int, pageSize : Int, totalResults : Int, chatLines : List[ChatLine]) { + page : Int, pageSize : Int, totalResults : Int, chatLines : Seq[ChatLine]) { def toSimpleSearchResult = SimpleSearchResult(server, channel, botName, query, page, pageSize, totalResults, chatLines map { case mline@ChatLine(_, _, _, contextBefore, contextAfter) => ((contextBefore :+ mline) ++ contextAfter) map { line => - List(line.timestamp.toString, line.user, line.message) + Seq(line.timestamp.toString, line.user, line.message) } }) } @@ -36,12 +36,12 @@ case class SearchResult( object SearchResult { def fromSearchRequest(searchRequest : SearchRequest) = searchRequest match { case SearchRequest(server, channel, botName, query, page, pageSize, _) => - new SearchResult(server, channel, botName, query, page, pageSize, 0, List()) + new SearchResult(server, channel, botName, query, page, pageSize, 0, Seq()) } } case class SimpleSearchResult( server : String, channel : String, botName : String, query: String, - page : Int, pageSize : Int, totalResults : Int, lines : List[List[List[String]]]) + page : Int, pageSize : Int, totalResults : Int, lines : Seq[Seq[Seq[String]]]) case class SearchError(error : String) \ No newline at end of file