abhin4v/irc-search
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Some more scalafication

Browse Source
This commit is contained in:
Abhinav Sarkar 2013-05-23 10:26:34 +05:30
parent 69d031461b
commit 75004747b3
5 changed files with 195 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pom.xml
View File

@ -1,4 +1,5 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>net.abhinavsarkar.ircsearch</groupId> <groupId>net.abhinavsarkar.ircsearch</groupId>

26
src/main/scala/net/abhinavsarkar/ircsearch/Server.scala
View File

@ -91,12 +91,12 @@ private object UnifiedHandler extends ChannelInboundByteHandlerAdapter {
} }
override def inboundBufferUpdated(ctx : ChannelHandlerContext, in: ByteBuf) { override def inboundBufferUpdated(ctx : ChannelHandlerContext, in: ByteBuf) {
if (in.readableBytes() < 5) { if (in.readableBytes < 5) {
return; return;
} }
val magic1 = in.getUnsignedByte(in.readerIndex()) val magic1 = in.getUnsignedByte(in.readerIndex)
val magic2 = in.getUnsignedByte(in.readerIndex() + 1) val magic2 = in.getUnsignedByte(in.readerIndex + 1)
if (isHttp(magic1, magic2)) { if (isHttp(magic1, magic2)) {
ctx.pipeline ctx.pipeline
.addLast("decoder", new HttpRequestDecoder) .addLast("decoder", new HttpRequestDecoder)
@ -107,7 +107,7 @@ private object UnifiedHandler extends ChannelInboundByteHandlerAdapter {
.remove(this) .remove(this)
} else { } else {
ctx.pipeline ctx.pipeline
.addLast("framedecoder", new DelimiterBasedFrameDecoder(1048576, Delimiters.lineDelimiter() : _*)) .addLast("framedecoder", new DelimiterBasedFrameDecoder(1048576, Delimiters.lineDelimiter : _*))
.addLast("decoder", new StringDecoder(Charset.forName("UTF-8"))) .addLast("decoder", new StringDecoder(Charset.forName("UTF-8")))
.addLast("csvhandler", new TcpIndexHandler) .addLast("csvhandler", new TcpIndexHandler)
.remove(this) .remove(this)
@ -147,7 +147,7 @@ private class TcpIndexHandler extends ChannelInboundMessageHandlerAdapter[String
inited = true inited = true
} else { } else {
Indexer.index(IndexRequest(server, channel, botName, Indexer.index(IndexRequest(server, channel, botName,
List(ChatLine(values(0), values(1).toLong, values(2))))) Seq(ChatLine(values(0), values(1).toLong, values(2)))))
} }
} }
} }
@ -155,7 +155,7 @@ private class TcpIndexHandler extends ChannelInboundMessageHandlerAdapter[String
@Sharable @Sharable
private object EchoHandler extends HttpRequestHandler { private object EchoHandler extends HttpRequestHandler {
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) { override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
val content = request.getContent().toString(Charset.forName("UTF-8")) val content = request.getContent.toString(Charset.forName("UTF-8"))
logRequest(ctx, request, sendSuccess(ctx, request, content)) logRequest(ctx, request, sendSuccess(ctx, request, content))
} }
} }
@ -165,7 +165,7 @@ private class IndexHandler extends HttpRequestHandler {
implicit val formats = DefaultFormats implicit val formats = DefaultFormats
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) { override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
future { future {
val content = request.getContent().toString(Charset.forName("UTF-8")) val content = request.getContent.toString(Charset.forName("UTF-8"))
val indexRequest = Serialization.read[IndexRequest](content) val indexRequest = Serialization.read[IndexRequest](content)
Indexer.index(indexRequest) Indexer.index(indexRequest)
} }
@ -178,16 +178,16 @@ private object SearchHandler extends HttpRequestHandler {
implicit val formats = DefaultFormats implicit val formats = DefaultFormats
override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) { override def messageReceived(ctx: ChannelHandlerContext, request: HttpRequest) {
val f = future { val f = future {
val method = request.getMethod() val method = request.getMethod
val searchRequest = if (HttpMethod.POST.equals(method)) { val searchRequest = if (HttpMethod.POST.equals(method)) {
val content = request.getContent().toString(Charset.forName("UTF-8")) val content = request.getContent.toString(Charset.forName("UTF-8"))
Serialization.read[SearchRequest](content) Serialization.read[SearchRequest](content)
} else if (HttpMethod.GET.equals(method)) { } else if (HttpMethod.GET.equals(method)) {
val params = new QueryStringDecoder(request.getUri).getParameters.toMap val params = new QueryStringDecoder(request.getUri).getParameters.toMap
val List(server, channel, botName, query) = val Seq(server, channel, botName, query) =
List("server", "channel", "botName", "query").map(params(_).get(0)) Seq("server", "channel", "botName", "query").map(params(_).get(0))
val List(page, pageSize, details) = val Seq(page, pageSize, details) =
List("page", "pageSize", "details").map(params.get(_).map({ case l => l.get(0) })) Seq("page", "pageSize", "details").map(params.get(_).map({ case l => l.get(0) }))
var sr = SearchRequest(server, channel, botName, query) var sr = SearchRequest(server, channel, botName, query)
if (page.isDefined) if (page.isDefined)

10
src/main/scala/net/abhinavsarkar/ircsearch/lucene/Indexer.scala
View File

@ -136,10 +136,10 @@ object Indexer extends Logging {
rec.chatLine.copy( rec.chatLine.copy(
contextBefore = recs.slice(idx - ContextSize, idx).map(_.chatLine) contextBefore = recs.slice(idx - ContextSize, idx).map(_.chatLine)
.filter(_.timestamp >= rec.chatLine.timestamp - ContextDurationSecs * 1000) .filter(_.timestamp >= rec.chatLine.timestamp - ContextDurationSecs * 1000)
.toList, .toSeq,
contextAfter = recs.slice(idx + 1, 2 * ContextSize + 1).map(_.chatLine) contextAfter = recs.slice(idx + 1, 2 * ContextSize + 1).map(_.chatLine)
.filter(_.timestamp <= rec.chatLine.timestamp + ContextDurationSecs * 1000) .filter(_.timestamp <= rec.chatLine.timestamp + ContextDurationSecs * 1000)
.toList)) .toSeq))
} }
def start { def start {
@ -148,7 +148,7 @@ object Indexer extends Logging {
if (!indexQueue.isEmpty) { if (!indexQueue.isEmpty) {
val indexRecs = new ArrayList[IndexRecord] val indexRecs = new ArrayList[IndexRecord]
indexQueue drainTo indexRecs indexQueue drainTo indexRecs
val indexRecsMap = indexRecs groupBy { r => (r.server, r.channel, r.botName) } val indexRecsMap = indexRecs.toIndexedSeq groupBy { r => (r.server, r.channel, r.botName) }
val windowSize = 2 * ContextSize + 1 val windowSize = 2 * ContextSize + 1
for (indexRecBatch <- indexRecsMap.values) { for (indexRecBatch <- indexRecsMap.values) {
@ -192,7 +192,7 @@ object Indexer extends Logging {
} }
} }
private def ctxToStr(ctx : List[ChatLine]) = private def ctxToStr(ctx : Seq[ChatLine]) =
ctx.map { line => s"${line.timestamp} ${line.user}: ${line.message}" } mkString "\n" ctx.map { line => s"${line.timestamp} ${line.user}: ${line.message}" } mkString "\n"
private def doIndex(indexRecord: IndexRecord) { private def doIndex(indexRecord: IndexRecord) {
@ -205,7 +205,7 @@ object Indexer extends Logging {
val msg = new TextField(ChatLine.MSG, chatLine.message, Field.Store.YES) val msg = new TextField(ChatLine.MSG, chatLine.message, Field.Store.YES)
val ctxBfr = new TextField(ChatLine.CTXB, ctxToStr(chatLine.contextBefore), Field.Store.YES) val ctxBfr = new TextField(ChatLine.CTXB, ctxToStr(chatLine.contextBefore), Field.Store.YES)
val ctxAft = new TextField(ChatLine.CTXA, ctxToStr(chatLine.contextAfter), Field.Store.YES) val ctxAft = new TextField(ChatLine.CTXA, ctxToStr(chatLine.contextAfter), Field.Store.YES)
indexWriter.addDocument(List(ts, user, msg, ctxBfr, ctxAft), indexWriter.getAnalyzer) indexWriter.addDocument(Seq(ts, user, msg, ctxBfr, ctxAft), indexWriter.getAnalyzer)
logger.debug("Indexed : [{} {} {}] [{}] {}: {}", logger.debug("Indexed : [{} {} {}] [{}] {}: {}",
server, channel, botName, new Date(chatLine.timestamp), chatLine.user, chatLine.message) server, channel, botName, new Date(chatLine.timestamp), chatLine.user, chatLine.message)
} }

14
src/main/scala/net/abhinavsarkar/ircsearch/lucene/Searcher.scala
View File

@ -51,7 +51,7 @@ object Searcher extends Logging {
private def mkQueryParser(analyzer : Analyzer) = private def mkQueryParser(analyzer : Analyzer) =
new MultiFieldQueryParser(Indexer.LuceneVersion, new MultiFieldQueryParser(Indexer.LuceneVersion,
List(ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA).toArray, analyzer, Array(ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA), analyzer,
Map(ChatLine.MSG -> MessageFieldBoost)) Map(ChatLine.MSG -> MessageFieldBoost))
private def filterifyQuery(query : Query) : Query = private def filterifyQuery(query : Query) : Query =
@ -121,13 +121,13 @@ object Searcher extends Logging {
} }
} }
private val DocFields = List(ChatLine.USER, ChatLine.TS, ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA) private val DocFields = Seq(ChatLine.USER, ChatLine.TS, ChatLine.MSG, ChatLine.CTXB, ChatLine.CTXA)
private def doSearch(indexDir : String, query : Query, page : Int, pageSize : Int) private def doSearch(indexDir : String, query : Query, page : Int, pageSize : Int)
: (Int, List[(ChatLine, Float)]) = { : (Int, Seq[(ChatLine, Float)]) = {
val searcherMgr = getSearcherMgr(indexDir) val searcherMgr = getSearcherMgr(indexDir)
searcherMgr.maybeRefresh searcherMgr.maybeRefresh
val indexSearcher = searcherMgr.acquire() val indexSearcher = searcherMgr.acquire
try { try {
val topDocs = indexSearcher.search(query, MaxHits.min((page + 1) * pageSize), val topDocs = indexSearcher.search(query, MaxHits.min((page + 1) * pageSize),
new Sort(SortField.FIELD_SCORE, new SortField(ChatLine.TS, SortField.Type.LONG, true))) new Sort(SortField.FIELD_SCORE, new SortField(ChatLine.TS, SortField.Type.LONG, true)))
@ -139,15 +139,15 @@ object Searcher extends Logging {
(map, field) => map += (field.name -> field.stringValue) (map, field) => map += (field.name -> field.stringValue)
} }
val List(user, timestamp, message, contextBefore, contextAfter) = DocFields.map(doc) val Seq(user, timestamp, message, contextBefore, contextAfter) = DocFields.map(doc)
val LineRe = "(\\d+) (.*?): (.*)".r val LineRe = "(\\d+) (.*?): (.*)".r
val List(ctxBefore, ctxAfter) = List(contextBefore, contextAfter).map { val Seq(ctxBefore, ctxAfter) = Seq(contextBefore, contextAfter).map {
_.split('\n').filterNot(_.isEmpty).map { _.split('\n').filterNot(_.isEmpty).map {
case LineRe(timestamp, user, message) => ChatLine(user, timestamp.toLong, message) case LineRe(timestamp, user, message) => ChatLine(user, timestamp.toLong, message)
}} }}
val chatLine = ChatLine(user, timestamp.toLong, message, ctxBefore.toList, ctxAfter.toList) val chatLine = ChatLine(user, timestamp.toLong, message, ctxBefore, ctxAfter)
(chatLine, score) (chatLine, score)
} }
(topDocs.totalHits, docs.toList) (topDocs.totalHits, docs.toList)

14
src/main/scala/net/abhinavsarkar/ircsearch/model.scala
View File

@ -10,11 +10,11 @@ object ChatLine {
} }
case class ChatLine(user : String, timestamp : Long, message : String, case class ChatLine(user : String, timestamp : Long, message : String,
contextBefore : List[ChatLine] = List(), contextBefore : Seq[ChatLine] = Seq(),
contextAfter : List[ChatLine] = List()) contextAfter : Seq[ChatLine] = Seq())
case class IndexRequest( case class IndexRequest(
server : String, channel : String, botName : String, chatLines : List[ChatLine]) server : String, channel : String, botName : String, chatLines : Seq[ChatLine])
case class SearchRequest( case class SearchRequest(
server : String, channel : String, botName : String, query: String, server : String, channel : String, botName : String, query: String,
@ -22,13 +22,13 @@ case class SearchRequest(
case class SearchResult( case class SearchResult(
server : String, channel : String, botName : String, query: String, server : String, channel : String, botName : String, query: String,
page : Int, pageSize : Int, totalResults : Int, chatLines : List[ChatLine]) { page : Int, pageSize : Int, totalResults : Int, chatLines : Seq[ChatLine]) {
def toSimpleSearchResult = def toSimpleSearchResult =
SimpleSearchResult(server, channel, botName, query, page, pageSize, totalResults, SimpleSearchResult(server, channel, botName, query, page, pageSize, totalResults,
chatLines map { chatLines map {
case mline@ChatLine(_, _, _, contextBefore, contextAfter) => case mline@ChatLine(_, _, _, contextBefore, contextAfter) =>
((contextBefore :+ mline) ++ contextAfter) map { line => ((contextBefore :+ mline) ++ contextAfter) map { line =>
List(line.timestamp.toString, line.user, line.message) Seq(line.timestamp.toString, line.user, line.message)
} }
}) })
} }
@ -36,12 +36,12 @@ case class SearchResult(
object SearchResult { object SearchResult {
def fromSearchRequest(searchRequest : SearchRequest) = searchRequest match { def fromSearchRequest(searchRequest : SearchRequest) = searchRequest match {
case SearchRequest(server, channel, botName, query, page, pageSize, _) => case SearchRequest(server, channel, botName, query, page, pageSize, _) =>
new SearchResult(server, channel, botName, query, page, pageSize, 0, List()) new SearchResult(server, channel, botName, query, page, pageSize, 0, Seq())
} }
} }
case class SimpleSearchResult( case class SimpleSearchResult(
server : String, channel : String, botName : String, query: String, server : String, channel : String, botName : String, query: String,
page : Int, pageSize : Int, totalResults : Int, lines : List[List[List[String]]]) page : Int, pageSize : Int, totalResults : Int, lines : Seq[Seq[Seq[String]]])
case class SearchError(error : String) case class SearchError(error : String)