| 1 | (ns micro-blog.blue-sky
|
| 2 | (:require
|
| 3 | [clj-http.client :as http-client]
|
| 4 | [micro-blog.pocket-base :as pb]
|
| 5 | [micro-blog.utils :as utils]
|
| 6 | [micro-blog.is-tech]
|
| 7 | [taoensso.telemere :as tel]
|
| 8 | [micro-blog.config :refer [config]]))
|
| 9 |
|
| 10 | (defn create-session []
|
| 11 | (let [identifier (@config :blue-sky-username)
|
| 12 | api-key (@config :blue-sky-api-key)
|
| 13 | body {:identifier identifier :password api-key}
|
| 14 | url (str (@config :blue-sky-host) "/com.atproto.server.createSession")
|
| 15 | res-schema [:map
|
| 16 | [:did string?]
|
| 17 | [:accessJwt string?]]]
|
| 18 | (-> (http-client/post url
|
| 19 | {:form-params body
|
| 20 | :content-type :json
|
| 21 | :as :json})
|
| 22 | :body
|
| 23 | (utils/validate-with-throw res-schema)
|
| 24 | (#(assoc % :access-jwt (:accessJwt %)))
|
| 25 | (select-keys [:did :access-jwt]))))
|
| 26 |
|
| 27 | (def post-res-schema [:map
|
| 28 | [:cursor [:maybe :string]]
|
| 29 | [:feed [:vector
|
| 30 | [:map [:post [:map
|
| 31 | [:cid :string]
|
| 32 | [:author [:map
|
| 33 | [:handle :string]]]
|
| 34 |
|
| 35 | [:embed {:optional true}
|
| 36 | [:map [:images {:optional true} [:vector [:map
|
| 37 | [:fullsize :string]
|
| 38 | [:alt :string]]]]]]
|
| 39 | [:record [:map
|
| 40 | [:facets {:optional true} [:vector [:map
|
| 41 | [:features [:vector [:map
|
| 42 | [:$type :string]
|
| 43 | [:tag {:optional true} [:maybe :string]]]]]]]]
|
| 44 | [:createdAt :string]]]]]]]]])
|
| 45 |
|
| 46 | (defn get-posts-until-id
|
| 47 | ([session id] (get-posts-until-id session id nil []))
|
| 48 | ([session id cursor prev-posts]
|
| 49 | (tel/log! {:level :info :data {:postId (:remoteId id)}} "Getting posts until id")
|
| 50 | (let [limit 5
|
| 51 | body
|
| 52 | (-> (http-client/get (str (@config :blue-sky-host) "/app.bsky.feed.getAuthorFeed")
|
| 53 | {:headers {"Authorization" (str "Bearer " (session :access-jwt))}
|
| 54 | :query-params (cond-> {:actor (:did session)
|
| 55 | :limit limit}
|
| 56 | cursor (assoc :cursor cursor))
|
| 57 | :content-type :json
|
| 58 | :as :json})
|
| 59 | :body
|
| 60 | (utils/validate-with-throw post-res-schema))
|
| 61 | posts (map :post (:feed body))
|
| 62 | new-cursor (:cursor body)
|
| 63 | new-posts (take-while #(not= (:cid %) id) posts)
|
| 64 | new-and-prev-posts (concat new-posts prev-posts)]
|
| 65 | (cond
|
| 66 | ;; end of posts
|
| 67 | (not= (count posts) limit) new-and-prev-posts
|
| 68 | ;; found post
|
| 69 | (some #(= id (:cid %)) posts) new-and-prev-posts
|
| 70 | ;; recur
|
| 71 | :else (recur session id new-cursor new-and-prev-posts)))))
|
| 72 |
|
| 73 | (defn extract-tags [post]
|
| 74 | (let [facets (get (post :record) :facets [])
|
| 75 | features (flatten (map :features facets))
|
| 76 | tag-features (filter #(= (:$type %) "app.bsky.richtext.facet#tag") features)
|
| 77 | tags (map :tag tag-features)]
|
| 78 | tags))
|
| 79 |
|
| 80 | (defn extract-images [post]
|
| 81 | (let [images (get-in post [:embed :images] [])]
|
| 82 | (map #(vector (:fullsize %) (:alt %)) images)))
|
| 83 |
|
| 84 | (defn transform-post [post]
|
| 85 | (hash-map :source :blue_sky
|
| 86 | :fullPost post
|
| 87 | :remoteId (:cid post)
|
| 88 | :isTech (micro-blog.is-tech/is-tech? (:record post))
|
| 89 | :authorId (get-in post [:author :handle])
|
| 90 | :tags (extract-tags post)
|
| 91 | :images (extract-images post)
|
| 92 | :posted (get-in post [:record :createdAt])))
|
| 93 |
|
| 94 | (defn save-post [post]
|
| 95 | (tel/log! {:level :info :data {:postId (:remoteId post)}} "Saving post")
|
| 96 | (pb/save-post post))
|
| 97 |
|
| 98 | (defn run []
|
| 99 | (tel/log! :info "Running blue sky fetcher")
|
| 100 | (let [session (create-session)
|
| 101 | last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky)
|
| 102 | new-posts (reverse (get-posts-until-id session last-saved-id))]
|
| 103 | (doseq [post new-posts]
|
| 104 | (-> post transform-post save-post))))
|
| 105 |
|
