travisshears hat die Gist bearbeitet . Zu Änderung gehen
1 file changed, 104 insertions
blue_sky.clj(Datei erstellt)
| @@ -0,0 +1,104 @@ | |||
| 1 | + | (ns micro-blog.blue-sky | |
| 2 | + | (:require | |
| 3 | + | [clj-http.client :as http-client] | |
| 4 | + | [micro-blog.pocket-base :as pb] | |
| 5 | + | [micro-blog.utils :as utils] | |
| 6 | + | [micro-blog.is-tech] | |
| 7 | + | [taoensso.telemere :as tel] | |
| 8 | + | [micro-blog.config :refer [config]])) | |
| 9 | + | ||
| 10 | + | (defn create-session [] | |
| 11 | + | (let [identifier (@config :blue-sky-username) | |
| 12 | + | api-key (@config :blue-sky-api-key) | |
| 13 | + | body {:identifier identifier :password api-key} | |
| 14 | + | url (str (@config :blue-sky-host) "/com.atproto.server.createSession") | |
| 15 | + | res-schema [:map | |
| 16 | + | [:did string?] | |
| 17 | + | [:accessJwt string?]]] | |
| 18 | + | (-> (http-client/post url | |
| 19 | + | {:form-params body | |
| 20 | + | :content-type :json | |
| 21 | + | :as :json}) | |
| 22 | + | :body | |
| 23 | + | (utils/validate-with-throw res-schema) | |
| 24 | + | (#(assoc % :access-jwt (:accessJwt %))) | |
| 25 | + | (select-keys [:did :access-jwt])))) | |
| 26 | + | ||
| 27 | + | (def post-res-schema [:map | |
| 28 | + | [:cursor [:maybe :string]] | |
| 29 | + | [:feed [:vector | |
| 30 | + | [:map [:post [:map | |
| 31 | + | [:cid :string] | |
| 32 | + | [:author [:map | |
| 33 | + | [:handle :string]]] | |
| 34 | + | ||
| 35 | + | [:embed {:optional true} | |
| 36 | + | [:map [:images {:optional true} [:vector [:map | |
| 37 | + | [:fullsize :string] | |
| 38 | + | [:alt :string]]]]]] | |
| 39 | + | [:record [:map | |
| 40 | + | [:facets {:optional true} [:vector [:map | |
| 41 | + | [:features [:vector [:map | |
| 42 | + | [:$type :string] | |
| 43 | + | [:tag {:optional true} [:maybe :string]]]]]]]] | |
| 44 | + | [:createdAt :string]]]]]]]]]) | |
| 45 | + | ||
| 46 | + | (defn get-posts-until-id | |
| 47 | + | ([session id] (get-posts-until-id session id nil [])) | |
| 48 | + | ([session id cursor prev-posts] | |
| 49 | + | (tel/log! {:level :info :data {:postId (:remoteId id)}} "Getting posts until id") | |
| 50 | + | (let [limit 5 | |
| 51 | + | body | |
| 52 | + | (-> (http-client/get (str (@config :blue-sky-host) "/app.bsky.feed.getAuthorFeed") | |
| 53 | + | {:headers {"Authorization" (str "Bearer " (session :access-jwt))} | |
| 54 | + | :query-params (cond-> {:actor (:did session) | |
| 55 | + | :limit limit} | |
| 56 | + | cursor (assoc :cursor cursor)) | |
| 57 | + | :content-type :json | |
| 58 | + | :as :json}) | |
| 59 | + | :body | |
| 60 | + | (utils/validate-with-throw post-res-schema)) | |
| 61 | + | posts (map :post (:feed body)) | |
| 62 | + | new-cursor (:cursor body) | |
| 63 | + | new-posts (take-while #(not= (:cid %) id) posts) | |
| 64 | + | new-and-prev-posts (concat new-posts prev-posts)] | |
| 65 | + | (cond | |
| 66 | + | ;; end of posts | |
| 67 | + | (not= (count posts) limit) new-and-prev-posts | |
| 68 | + | ;; found post | |
| 69 | + | (some #(= id (:cid %)) posts) new-and-prev-posts | |
| 70 | + | ;; recur | |
| 71 | + | :else (recur session id new-cursor new-and-prev-posts))))) | |
| 72 | + | ||
| 73 | + | (defn extract-tags [post] | |
| 74 | + | (let [facets (get (post :record) :facets []) | |
| 75 | + | features (flatten (map :features facets)) | |
| 76 | + | tag-features (filter #(= (:$type %) "app.bsky.richtext.facet#tag") features) | |
| 77 | + | tags (map :tag tag-features)] | |
| 78 | + | tags)) | |
| 79 | + | ||
| 80 | + | (defn extract-images [post] | |
| 81 | + | (let [images (get-in post [:embed :images] [])] | |
| 82 | + | (map #(vector (:fullsize %) (:alt %)) images))) | |
| 83 | + | ||
| 84 | + | (defn transform-post [post] | |
| 85 | + | (hash-map :source :blue_sky | |
| 86 | + | :fullPost post | |
| 87 | + | :remoteId (:cid post) | |
| 88 | + | :isTech (micro-blog.is-tech/is-tech? (:record post)) | |
| 89 | + | :authorId (get-in post [:author :handle]) | |
| 90 | + | :tags (extract-tags post) | |
| 91 | + | :images (extract-images post) | |
| 92 | + | :posted (get-in post [:record :createdAt]))) | |
| 93 | + | ||
| 94 | + | (defn save-post [post] | |
| 95 | + | (tel/log! {:level :info :data {:postId (:remoteId post)}} "Saving post") | |
| 96 | + | (pb/save-post post)) | |
| 97 | + | ||
| 98 | + | (defn run [] | |
| 99 | + | (tel/log! :info "Running blue sky fetcher") | |
| 100 | + | (let [session (create-session) | |
| 101 | + | last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky) | |
| 102 | + | new-posts (reverse (get-posts-until-id session last-saved-id))] | |
| 103 | + | (doseq [post new-posts] | |
| 104 | + | (-> post transform-post save-post)))) | |
travisshears hat die Gist bearbeitet . Zu Änderung gehen
Keine Änderungen
travisshears hat die Gist bearbeitet . Zu Änderung gehen
1 file changed, 25 insertions
is_tech.clj(Datei erstellt)
| @@ -0,0 +1,25 @@ | |||
| 1 | + | (ns micro-blog.is-tech | |
| 2 | + | (:require | |
| 3 | + | [micro-blog.config :refer [config]] | |
| 4 | + | [taoensso.telemere :as tel] | |
| 5 | + | [clj-http.client :as client])) | |
| 6 | + | ||
| 7 | + | (defn is-tech? [post-text] | |
| 8 | + | (let [url (str (:mistral-host @config) "/v1/conversations") | |
| 9 | + | headers {"Content-Type" "application/json" | |
| 10 | + | "Accept" "application/json" | |
| 11 | + | "Authorization" (str "Bearer " (@config :mistral-api-key))} | |
| 12 | + | body {:inputs post-text | |
| 13 | + | :stream false | |
| 14 | + | :agent_id (@config :mistral-agent-id)}] | |
| 15 | + | (tel/log! {:level :info :data {:url url :agent_id (:agent_id body)}} "making request to mistral agent") | |
| 16 | + | (-> | |
| 17 | + | (client/post url {:headers headers | |
| 18 | + | :form-params body | |
| 19 | + | :content-type :json | |
| 20 | + | :as :json}) | |
| 21 | + | :body | |
| 22 | + | :outputs | |
| 23 | + | first | |
| 24 | + | :content | |
| 25 | + | (#(if (= "1" %) true false))))) | |
Neuer
Älter