(def html-data (-> "https://stockton.craigslist.org/search/apa?max_price=3000"
                   java.net.URL.
                   enlive/html-resource))

; Pull out just the rows of results from all of the HTML
(defn select-rows [html-data]
  (enlive/select html-data [:p.result-info]))

(def rows (select-rows html-data))

; Helper to pull out the first match of an Enlive select
(def select-first (comp first enlive/select))

; Take in a single row and a map onto which to assoc the extracted date 
(defn row-post-date [row-data row]
  (let [post-date (-> (select-first row-data [:time]) :attrs :datetime)]
    (assoc row
           :post-date post-date)))

(row-post-date (first rows) {}) ; possible output: {:post-date "2016-03-18 12:26"}

; Same format as above, but assoc in the row's price, if it's valid
(defn row-price [row-data row]
  (let [price-str (-> (select-first row-data [:span.result-price])
                      :content
                      first)
        valid? (re-matches #"\$\d+" (or price-str ""))
        price (when valid?
                (Integer/parseInt (subs price-str 1)))]
    (assoc row
           :price price)))

(row-price (first rows) {}) ; possible output: {:price 2300}

(reduce #(when %1
           (%2 rows %1))
        {} ; Starting with an empty map
        [row-link row-post-date row-price row-where]) ; All of the extractors

; Example simulation of data:
; 1. Reduce calls the first extractor with the empty map
(row-link {})
  => {:link "http://craigslist.com/foo/blah"} ; output

; 2. The result of that is then collected and the next extractor is called
(row-post-date {:link "http://craigslist.com/foo/blah"})
  => {:link "http://craigslist.com/foo/blah"
      :post-date "Feb 2 of the 5th ice age"}

; 3. Rinse and repeat
(row-price {:link "http://craigslist.com/foo/blah"
            :post-date "Feb 2 of the 5th ice age"})
  => {:link "http://craigslist.com/foo/blah"
      :post-date "Feb 2 of the 5th ice age"
      :price 2300}

(row-where {:link "http://craigslist.com/foo/blah"
            :post-date "Feb 2 of the 5th ice age"
            :price 2300})
  => {:link "http://craigslist.com/foo/blah"
      :post-date "Feb 2 of the 5th ice age"
      :price 2300
      :where "sacramento"}

; The last result is the return value of the reduction. It's the map which
; describes the whole row, based on the extractors. It's the apartment data!

(def nick "padwatch")
(def channel "#padwatch")

(def connection (atom nil))

; By default, logs go to stdout; use this to quiet them
(defn eat-log [& args]
  (comment pprint args))

(defn disconnect! []
  (when @connection
    (swap! connection irc/kill)))

(defn connect! []
  (disconnect!)
  (reset! connection (irc/connect "irc.freenode.net"
                                  6667 nick
                                  :callbacks {:raw-log eat-log}))
  (irc/join @connection channel))

(defn message! [msg]
  (irc/message @connection channel msg))

; Take a url and return the shortened one, if possible
(defn shorten-url [url]
  (when url
    (try
      (slurp (str "http://tinyurl.com/api-create.php?url=" url))
      (catch Throwable _ ; tinyurl can time out; just skip the shortening
        url))))

(defn message-row! [row-info]
  (let [useful (merge (select-keys row-info [:where :style :price :sqft])
                      {:url (-> row-info :url shorten-url)
                       :walkscore (-> (:walkscore row-info)
                                      (update :url shorten-url)
                                      (dissoc :description))})]
    (irc/message @connection channel (pr-str useful))))

(let [min-price 1500 max-price 3000
      min-sqft 600 max-sqft 1000
      bedrooms 1 bathrooms 1
      min-walkscore 60]
  {:min-price min-price :max-price max-price
   :min-sqft min-sqft :max-sqft max-sqft
   :bedrooms bedrooms :bathrooms bathrooms
   :min-walkscore min-walkscore
   :source {:craigslist (let [base-url "https://sfbay.craigslist.org"
                              max-rows 20]
                          {:source "craigslist"
                           :enabled? true
                           :base-url base-url
                           :search-url (str base-url "/search/apa")
                           :max-rows max-rows
                           :cycle-length-ms (* 1000 60 max-rows) ; Amortize minutes per row
                           :params {:min_price min-price :max_price max-price
                                    :minSqft min-sqft :maxSqft max-sqft
                                    :bedrooms bedrooms :bathrooms bathrooms
                                    :no_smoking 1
                                    :availabilityMode 0 ; All dates
                                    :hasPic 1
                                    :sort "date"
                                    :search_distance 3
                                    :postal 94010
                                    :bundleDuplicates 0}})
            :zillow (let [zones ["belmont-ca" "half-moon-bay-ca"
                                 "highlands-baywood-park-ca" "hillsborough-ca"
                                 "pacifica-ca" "san-bruno-ca"
                                 "south-san-francisco-ca" "brisbane-ca"
                                 "daly-city-ca" "colma-ca" "millbrae-ca"
                                 "oakland-ca" "san-francisco-ca" "piedmont-ca"
                                 "berkeley-ca" "albany-ca" "richmond-ca"
                                 "emeryville-ca"
                                 "burlingame-ca" "san-mateo-ca"]]
                      {:source "zillow"
                       :enabled? false
                       :base-url "http://www.zillow.com"
                       :cycle-length-ms (* 1000 60 10 (count zones)) ; Amortize minutes per zone
                       :zones zones})
            :walkscore {:key "XXX-FILL-THIS-IN-XXX"
                        :format-url "http://api.walkscore.com/score?format=json&lat=%s&lon=%s&wsapikey=%s"}
            :census-geo {:format-url "https://geocoding.geo.census.gov/geocoder/geographies/onelineaddress?address=%s&benchmark=4&vintage=4"}}})