commit d8f2671e4127e9577e4eca1f6131c986ff45ad7e Author: Abhinav Sarkar Date: Thu Sep 2 00:44:54 2010 +0530 First commit. Got everything working diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bfcb76e --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +pom.xml +*jar +lib +classes +target \ No newline at end of file diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..568b500 --- /dev/null +++ b/README.MD @@ -0,0 +1,24 @@ +# clj-twitter-feelings + +Shows how people on twitter are feeling, in real-time. +Meant to be an example of a swing app in Clojure. + +## Usage + +Download the standalone jar and run it like: + + java -jar clj-twitter-feelings-1.0.0-standalone.jar + +## How it works + +* Access the twitter sample tweet stream +* Find the feeling related adjectives in the tweet status and find their type +(Positive, Neutral, Negative) +* Keep the count of the adjective types in a sliding windows of tweets +* Show the count on the UI + +## License + +Copyright (C) 2010 Abhinav Sarkar + +Distributed under the Eclipse Public License, the same as Clojure. diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..fe9bc5a --- /dev/null +++ b/project.clj @@ -0,0 +1,13 @@ +(defproject clj-twitter-feelings "1.0.0" + :description "How is Twitter feeling now?" + :dependencies [[org.clojure/clojure "1.2.0"] + [org.clojure/clojure-contrib "1.2.0"] + [org.apache.httpcomponents/httpclient "4.0.1"] + [net.sf.squirrel-sql.thirdparty-non-maven/substance "5.2_01"] + [com.miglayout/miglayout "3.7.3"] + [jfree/jfreechart "1.0.12"]] + :dev-dependencies [[leiningen-run "0.2"]] + :main clj-twitter-feelings.ui + :aot [clj-twitter-feelings.core] + :warn-on-reflection true + :jar-dir "target") \ No newline at end of file diff --git a/resources/clj_twitter_adjectives/adjectives/negative.txt b/resources/clj_twitter_adjectives/adjectives/negative.txt new file mode 100644 index 0000000..471ec0c --- /dev/null +++ b/resources/clj_twitter_adjectives/adjectives/negative.txt @@ -0,0 +1,93 @@ +absurd +afraid +angry +annoyed +anxious +arrogant +ashamed +awful +bad +bewildered +bloody +bored +broken +cloudy +concerned +condemned +confused +crazy +creepy +cruel +damaged +dangerous +dark +dead +defeated +defiant +depressed +difficult +disgusted +disturbed +doubtful +drab +dull +eerie +embarrassed +envious +evil +fake +false +fearful +fierce +flawed +filthy +foolish +frantic +frightened +grieving +grotesque +guilty +helpless +hopeless +hungry +hurt +ill +infamous +jealous +lonely +mad +naughty +nervous +obnoxious +outrageous +panicky +painful +poor +rancid +repulsive +safe +scared +scary +shy +shitty +sick +sleepy +sore +strange +stupid +tame +tense +terrible +tired +troubled +ugly +unsightly +unusual +upset +uptight +vain +weary +wicked +wild +worried +wrong \ No newline at end of file diff --git a/resources/clj_twitter_adjectives/adjectives/neutral.txt b/resources/clj_twitter_adjectives/adjectives/neutral.txt new file mode 100644 index 0000000..a8a3d6d --- /dev/null +++ b/resources/clj_twitter_adjectives/adjectives/neutral.txt @@ -0,0 +1,38 @@ +alert +alright +average +blushing +busy +calm +careful +cautious +concerned +crowded +curious +different +dirty +distinct +fair +fine +fragile +glamorous +important +impossible +inquisitive +light +misty +motionless +muddy +plain +pleasant +puzzled +shiny +shy +sleepy +smoggy +sparkling +spotless +stormy +strange +thick +tired \ No newline at end of file diff --git a/resources/clj_twitter_adjectives/adjectives/positive.txt b/resources/clj_twitter_adjectives/adjectives/positive.txt new file mode 100644 index 0000000..76a1a57 --- /dev/null +++ b/resources/clj_twitter_adjectives/adjectives/positive.txt @@ -0,0 +1,92 @@ +adorable +agreeable +alert +alive +amused +amazing +awesome +beautiful +brainy +brave +bright +bright +caring +charming +cheerful +clean +clear +clever +colourful +comfortable +cool +cooperative +courageous +cute +delightful +determined +eager +easy +elated +elegant +enchanting +encouraging +energetic +enthusiastic +excited +exuberant +excellent +faithful +famous +fancy +fantastic +forward +free +friendly +frowning +funny +gentle +gleaming +glorious +good +gorgeous +graceful +happy +healthy +helpful +hilarious +homely +innocent +jolly +kind +lively +lovely +lucky +modern +obedient +open +outgoing +outstanding +perfect +poised +positive +powerful +proud +quaint +real +relaxed +relieved +rich +right +robust +sane +silly +smiling +splendid +successful +super +thoughtful +victorious +vivacious +well +witty +wonderful \ No newline at end of file diff --git a/resources/clj_twitter_adjectives/favicon.jpg b/resources/clj_twitter_adjectives/favicon.jpg new file mode 100644 index 0000000..b013752 Binary files /dev/null and b/resources/clj_twitter_adjectives/favicon.jpg differ diff --git a/src/clj_twitter_feelings/core.clj b/src/clj_twitter_feelings/core.clj new file mode 100644 index 0000000..a15d964 --- /dev/null +++ b/src/clj_twitter_feelings/core.clj @@ -0,0 +1,119 @@ +(ns clj-twitter-feelings.core + (:import [java.io File BufferedReader] + [clojure.lang PersistentQueue] + [org.apache.http HttpException] + [org.apache.http.auth AuthScope UsernamePasswordCredentials] + [org.apache.http.client.methods HttpGet] + [org.apache.http.client ResponseHandler HttpClient] + [org.apache.http.impl.client DefaultHttpClient] + [org.apache.http.params BasicHttpParams HttpParams]) + (:use [clojure.java.io :only (reader resource as-file)] + [clojure.contrib + [core :only (-?>)] + [string :only (split lower-case)] + [duck-streams :only (read-lines)] + [json :only (read-json)]])) + +(defmulti trace (fn [_ f _] f)) +(defmethod trace :l [msg _ arg] (do (println msg ":" arg) arg)) +(defmethod trace :f [arg _ msg] (do (println msg ":" arg) arg)) + +(def adjective-files ["negative" "neutral" "positive"]) + +(defn adjectives [] + (->> adjective-files + (map #(str "clj_twitter_adjectives/adjectives/" % ".txt")) + (map resource) + (reduce + (fn [acc ^java.net.URL url] + (let [adjective-type + (-> url .toString (.split "/") last (.split "\\.") first)] + (reduce + (fn [acc word] + (assoc! acc word adjective-type)) + acc + (read-lines url)))) + (transient {})) + (persistent!))) + +(defn safe-divide [n d] (if (zero? d) 0 (float (/ n d)))) + +(def split-pattern (re-pattern "[\\p{Z}\\p{C}\\p{P}]+")) + +(defn tokenize-line [line] + (->> line (split split-pattern) (filter (complement empty?)))) + +(defprotocol Processor + (process [this tweet])) + +(defn twitter-stream-client [username password] + (doto (DefaultHttpClient.) + (.. getCredentialsProvider + (setCredentials + (AuthScope. "stream.twitter.com" 80) + (UsernamePasswordCredentials. username password))))) + +(defn tweet-stream [^HttpClient client method & params] + (let [read-line (fn this [^BufferedReader rdr] + (lazy-seq + (if-let [line (.readLine rdr)] + (cons line (this rdr)) + (.close rdr)))) + baseurl "http://stream.twitter.com/1/statuses/" + url (str baseurl method ".json") + http-params + (reduce (fn [^HttpParams hp [k v]] (.setParameter hp (name k) v)) + (BasicHttpParams.) (partition 2 params)) + request (doto (HttpGet. url) (.setParams http-params)) + response (.execute client request) + status-code (.. response getStatusLine getStatusCode)] + (if (= status-code 200) + (if-let [rdr (-?> response .getEntity .getContent reader)] + (map #(read-json % true) (read-line rdr))) + (throw (HttpException. + (str "Invalid Status code: " status-code)))))) + +(defn process-tweet-stream [stream processors] + (doseq [tweet stream] + (future (doseq [p processors] (process p tweet))))) + +(def status-seen (atom nil)) + +(defn status-processor [] + (reify Processor + (process [this tweet] + (reset! status-seen + (str (-> tweet :user :screen_name) ": " (:text tweet)))))) + +(def adjective-type-count (atom {})) + +(def adjective-seen (atom nil)) + +(def *tweet-window-size* 25) + +(defn adjective-processor [adjective-map] + (let [states (atom (PersistentQueue/EMPTY))] + (reify Processor + (process [this tweet] + (let [adj-typs + (->> tweet :text + tokenize-line + (map lower-case) + (map #(vector % (adjective-map %))) + (filter #(-> % second nil? not)) + ;(map #(do (println %) %)) + (map #(do (reset! adjective-seen (first %)) %)) + (map second))] + (when-not (empty? adj-typs) + (let [current-state + (reduce #(assoc %1 %2 (inc (get %1 %2 0))) {} adj-typs)] + (swap! adjective-type-count + (fn [state] + (if (<= (count @states) *tweet-window-size*) + (do (swap! states conj current-state) + (merge-with + state current-state)) + (let [old-state (peek @states)] + (swap! states #(conj (pop %) current-state)) + (merge-with #(max 0 (- %1 %2)) + (merge-with + state current-state) + old-state)))))))))))) diff --git a/src/clj_twitter_feelings/ui.clj b/src/clj_twitter_feelings/ui.clj new file mode 100644 index 0000000..35b6908 --- /dev/null +++ b/src/clj_twitter_feelings/ui.clj @@ -0,0 +1,225 @@ +(ns clj-twitter-feelings.ui + (:import [java.awt Dimension Color Font] + [java.awt.event KeyEvent KeyAdapter] + [javax.imageio ImageIO] + [javax.swing JPanel JFrame JLabel JDialog JTextField JPasswordField + JButton JOptionPane + Timer WindowConstants UIManager] + [org.jfree.chart ChartFactory ChartPanel] + [org.jfree.chart.plot PiePlot] + [org.jfree.chart.labels StandardPieSectionLabelGenerator] + [org.jfree.data.general DefaultPieDataset] + [org.jfree.data.time Millisecond TimeSeries TimeSeriesCollection] + [org.jfree.ui RefineryUtilities]) + (:use [clj-twitter-feelings.core] + [clojure.java.io :only (resource)] + [clojure.contrib + [miglayout :only (miglayout)] + [swing-utils :only (add-action-listener add-key-typed-listener + do-swing)]]) + (:gen-class)) + +(JFrame/setDefaultLookAndFeelDecorated true) +(JDialog/setDefaultLookAndFeelDecorated true) +(UIManager/setLookAndFeel + "org.jvnet.substance.skin.SubstanceModerateLookAndFeel") + +(let [message-type { + :error JOptionPane/ERROR_MESSAGE + :info JOptionPane/INFORMATION_MESSAGE + :warn JOptionPane/WARNING_MESSAGE + :question JOptionPane/QUESTION_MESSAGE + :plain JOptionPane/PLAIN_MESSAGE + }] + (defn show-message [frame message title type] + (JOptionPane/showMessageDialog frame message title (type message-type)))) + +(defn exit-app [^JFrame frame] + (doto frame (.setVisible false) (.dispose)) + (System/exit 0)) + +(defn create-auth-input-dialog + "Creates a JDialog to take the input of username and password from the user. + Returns the dialog. + + Arguments are: + + parent: the parent frame + dialog-title: the title of the dialog + dialog-message: the message shown in the dialog + dialog-width : the width of the dialog + dialog-height: the height of the dialog + username-lbl-text: the text shown on the username label + password-lbl-text: the text shown on the password label + input-field-size: the size of the username and password input fields + ok-btn-text: the text shown on the ok button + cancel-btn-text: the text shown on the cancel button + validation-fn: a function which is called to validate the user input when + the user presses the ok button. + the function is called with arguments: username, password, this dialog. + if the function return a string, it is shown on the dialog as the error + message and the dialog remains visible. otherwise ok-fn is called. + ok-fn: a function which is called when the user presses ok button and the + input is valid as per the call to validation-fn. + the function is called with arguments: username, password, this dialog. + the dialog is hidden before the call. + cancel-fn: a function which is called when the user presses cancel button. + the function is called with arguments: this dialog. + the dialog is hidden before the call. + " + [^JFrame parent + ^String dialog-title ^String dialog-message dialog-width dialog-height + ^String username-lbl-text ^String password-lbl-text input-field-size + ^String ok-btn-text ^String cancel-btn-text + validation-fn ok-fn cancel-fn] + (let [username-input (JTextField. (int input-field-size)) + password-input (JPasswordField. (int input-field-size)) + validation-msg-lbl (JLabel. " ") + ok-btn (JButton. ok-btn-text) + cancel-btn (JButton. cancel-btn-text) + dialog (JDialog. parent dialog-title true)] + (doseq [^JTextField in [username-input password-input]] + (.addKeyListener in + (proxy [KeyAdapter] [] + (keyTyped [^KeyEvent e] + (when (= (.getKeyChar e) \newline) + (.doClick ok-btn))) + (keyPressed [^KeyEvent e] + (when (= (.getKeyCode e) KeyEvent/VK_ESCAPE) + (.doClick cancel-btn)))))) + (doto dialog + (.setDefaultCloseOperation JDialog/DO_NOTHING_ON_CLOSE) + (.setContentPane + (miglayout (JPanel.) + :layout {:wrap 2} + (JLabel. dialog-message) {:span 2} + (JLabel. username-lbl-text) username-input + (JLabel. password-lbl-text) password-input + validation-msg-lbl {:span 2 :align "center"} + (miglayout (JPanel.) + (doto ok-btn + (add-action-listener + (fn [e] + (let [username (.getText username-input) + password (.getText password-input)] + (if-let [validation-msg + (validation-fn username password dialog)] + (.setText validation-msg-lbl validation-msg) + (do (.setText validation-msg-lbl " ") + (.setVisible dialog false) + (ok-fn username password dialog))))))) + (doto cancel-btn + (add-action-listener + (fn [e] + (.setVisible dialog false) + (cancel-fn dialog))))) + {:span 2 :align "center"})) + (.setSize dialog-width dialog-height)))) + +(defn init-gui [adjective-map] + (let [frame (JFrame. "Twitter Feelings") + + ^DefaultPieDataset pie-dataset + (reduce #(do (.setValue ^DefaultPieDataset %1 ^String %2 0) %1) + (DefaultPieDataset.) (sort (keys @adjective-type-count))) + pie-chart (ChartFactory/createPieChart + "Distribution" pie-dataset true false false) + pie-chart-panel (doto (ChartPanel. pie-chart) + (.setPreferredSize (Dimension. 500 400))) + + time-series-map + (into (sorted-map) + (map #(vector % (TimeSeries. % Millisecond)) + (keys @adjective-type-count))) + time-series-dataset + (reduce #(do (.addSeries ^TimeSeriesCollection %1 %2) %1) + (TimeSeriesCollection.) (vals time-series-map)) + time-series-chart + (ChartFactory/createTimeSeriesChart + "History" "Time" "Percentage" time-series-dataset + true false false) + time-series-chart-panel + (doto (ChartPanel. time-series-chart) + (.setPreferredSize (Dimension. 550 400))) + + adjective-lbl (JLabel. "

...

") + status-lbl (doto (JLabel. " ") + (.setFont (Font/getFont "Arial Unicode MS"))) + + timer (doto (Timer. 1000 nil) + (add-action-listener + (fn [e] + (let [a-count @adjective-type-count + total (reduce + 0 (vals a-count))] + (doseq [[^String k v] a-count] + (.setValue pie-dataset k (double v)) + (.add ^TimeSeries (time-series-map k) + (Millisecond.) (* 100 (safe-divide v total)))))))) + + ^JDialog auth-input-dialog + (create-auth-input-dialog frame + "Credentials" "Input your Twitter credentials" 220 150 + "Screen Name" "Password" 20 + "OK" "Cancel" + (fn [uname pass dialog] + (when (or (empty? uname) (empty? pass)) + (str "Please input Screen Name and Password"))) + (fn [uname pass ^JDialog dialog] + (future + (try + (process-tweet-stream + (tweet-stream (twitter-stream-client uname pass) "sample") + [(adjective-processor adjective-map) (status-processor)]) + (catch Exception e + (do-swing + (show-message frame + (str "Error happened: " (.getMessage e) + ".\nPlease restart.") + "Error" :error) + (.setVisible dialog true))))) + (.start timer)) + (fn [dialog] (exit-app frame)))] + (add-watch adjective-seen :adjective-lbl + (fn [_ _ _ n] + (do-swing + (.setText adjective-lbl (str "

" n "

"))))) + (add-watch status-seen :status-lbl + (fn [_ _ _ n] (do-swing (.setText status-lbl n)))) + (doto ^PiePlot (.getPlot pie-chart) + (.setNoDataMessage "No data available") + (.setLabelGenerator (StandardPieSectionLabelGenerator. "{0} {2}")) + (.setBackgroundPaint (Color. 238 238 238))) + (doto (.getXYPlot time-series-chart) + (.setBackgroundPaint (Color. 238 238 238))) + (doto (.. time-series-chart getXYPlot getDomainAxis) + (.setAutoRange true) + (.setFixedAutoRange 120000.0)) + (doto (.. time-series-chart getXYPlot getRangeAxis) + (.setRange 0.0 100.0)) + (doto frame + (.setIconImage + (ImageIO/read (resource "clj_twitter_adjectives/favicon.jpg"))) + (.setDefaultCloseOperation WindowConstants/EXIT_ON_CLOSE) + (.setResizable false) + (.setContentPane + (miglayout (JPanel.) + :layout [:wrap 1] + (miglayout (JPanel.) + :layout [:wrap 2] + (JLabel. "

How is Twitter feeling now?

") + {:align "left"} + adjective-lbl {:align "right"} + pie-chart-panel time-series-chart-panel) + status-lbl {:align "left"})) + (.pack) + (.setVisible true) + (RefineryUtilities/centerFrameOnScreen)) + (doto auth-input-dialog + (RefineryUtilities/centerFrameOnScreen) + (.setVisible true)))) + +(defn -main [& args] + (let [adjective-map (adjectives)] + (reset! adjective-type-count + (zipmap (distinct (vals adjective-map)) (repeat 0))) + (do-swing (init-gui adjective-map))))