travisshears / is_tech?

(ns micro-blog.blue-sky (:require [clj-http.client :as http-client] [micro-blog.pocket-base :as pb] [micro-blog.utils :as utils] [micro-blog.is-tech] [taoensso.telemere :as tel] [micro-blog.config :refer [config]])) (defn create-session [] (let [identifier (@config :blue-sky-username) api-key (@config :blue-sky-api-key) body {:identifier identifier :password api-key} url (str (@config :blue-sky-host) "/com.atproto.server.createSession") res-schema [:map [:did string?] [:accessJwt string?]]] (-> (http-client/post url {:form-params body :content-type :json :as :json}) :body (utils/validate-with-throw res-schema) (#(assoc % :access-jwt (:accessJwt %))) (select-keys [:did :access-jwt])))) (def post-res-schema [:map [:cursor [:maybe :string]] [:feed [:vector [:map [:post [:map [:cid :string] [:author [:map [:handle :string]]] [:embed {:optional true} [:map [:images {:optional true} [:vector [:map [:fullsize :string] [:alt :string]]]]]] [:record [:map [:facets {:optional true} [:vector [:map [:features [:vector [:map [:$type :string] [:tag {:optional true} [:maybe :string]]]]]]]] [:createdAt :string]]]]]]]]]) (defn get-posts-until-id ([session id] (get-posts-until-id session id nil [])) ([session id cursor prev-posts] (tel/log! {:level :info :data {:postId (:remoteId id)}} "Getting posts until id") (let [limit 5 body (-> (http-client/get (str (@config :blue-sky-host) "/app.bsky.feed.getAuthorFeed") {:headers {"Authorization" (str "Bearer " (session :access-jwt))} :query-params (cond-> {:actor (:did session) :limit limit} cursor (assoc :cursor cursor)) :content-type :json :as :json}) :body (utils/validate-with-throw post-res-schema)) posts (map :post (:feed body)) new-cursor (:cursor body) new-posts (take-while #(not= (:cid %) id) posts) new-and-prev-posts (concat new-posts prev-posts)] (cond ;; end of posts (not= (count posts) limit) new-and-prev-posts ;; found post (some #(= id (:cid %)) posts) new-and-prev-posts ;; recur :else (recur session id new-cursor new-and-prev-posts))))) (defn extract-tags [post] (let [facets (get (post :record) :facets []) features (flatten (map :features facets)) tag-features (filter #(= (:$type %) "app.bsky.richtext.facet#tag") features) tags (map :tag tag-features)] tags)) (defn extract-images [post] (let [images (get-in post [:embed :images] [])] (map #(vector (:fullsize %) (:alt %)) images))) (defn transform-post [post] (hash-map :source :blue_sky :fullPost post :remoteId (:cid post) :isTech (micro-blog.is-tech/is-tech? (:record post)) :authorId (get-in post [:author :handle]) :tags (extract-tags post) :images (extract-images post) :posted (get-in post [:record :createdAt]))) (defn save-post [post] (tel/log! {:level :info :data {:postId (:remoteId post)}} "Saving post") (pb/save-post post)) (defn run [] (tel/log! :info "Running blue sky fetcher") (let [session (create-session) last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky) new-posts (reverse (get-posts-until-id session last-saved-id))] (doseq [post new-posts] (-> post transform-post save-post))))

(ns micro-blog.is-tech (:require [micro-blog.config :refer [config]] [taoensso.telemere :as tel] [clj-http.client :as client])) (defn is-tech? [post-text] (let [url (str (:mistral-host @config) "/v1/conversations") headers {"Content-Type" "application/json" "Accept" "application/json" "Authorization" (str "Bearer " (@config :mistral-api-key))} body {:inputs post-text :stream false :agent_id (@config :mistral-agent-id)}] (tel/log! {:level :info :data {:url url :agent_id (:agent_id body)}} "making request to mistral agent") (-> (client/post url {:headers headers :form-params body :content-type :json :as :json}) :body :outputs first :content (#(if (= "1" %) true false)))))