travisshears/is_tech?

blue_sky.clj · 4.6 KiB · Clojure Raw

(ns micro-blog.blue-sky
  (:require
   [clj-http.client :as http-client]
   [micro-blog.pocket-base :as pb]
   [micro-blog.utils :as utils]
   [micro-blog.is-tech]
   [taoensso.telemere :as tel]
   [micro-blog.config :refer [config]]))

(defn create-session []
  (let [identifier (@config :blue-sky-username)
        api-key (@config :blue-sky-api-key)
        body {:identifier identifier :password api-key}
        url (str (@config :blue-sky-host) "/com.atproto.server.createSession")
        res-schema [:map
                    [:did string?]
                    [:accessJwt string?]]]
    (-> (http-client/post url
                          {:form-params body
                           :content-type :json
                           :as :json})
        :body
        (utils/validate-with-throw res-schema)
        (#(assoc % :access-jwt (:accessJwt %)))
        (select-keys  [:did :access-jwt]))))

(def post-res-schema [:map
                      [:cursor [:maybe :string]]
                      [:feed [:vector
                              [:map [:post [:map
                                            [:cid :string]
                                            [:author [:map
                                                      [:handle :string]]]

[:embed {:optional true}
                                             [:map [:images {:optional true} [:vector [:map
                                                                                       [:fullsize :string]
                                                                                       [:alt :string]]]]]]
                                            [:record [:map
                                                      [:facets {:optional true}  [:vector [:map
                                                                                           [:features [:vector [:map
                                                                                                                [:$type :string]
                                                                                                                [:tag {:optional true} [:maybe :string]]]]]]]]
                                                      [:createdAt :string]]]]]]]]])

(defn get-posts-until-id
  ([session id] (get-posts-until-id session id nil []))
  ([session id cursor prev-posts]
   (tel/log! {:level :info :data {:postId (:remoteId id)}} "Getting posts until id")
   (let [limit 5
         body
         (-> (http-client/get (str (@config :blue-sky-host) "/app.bsky.feed.getAuthorFeed")
                              {:headers {"Authorization" (str "Bearer " (session :access-jwt))}
                               :query-params  (cond-> {:actor (:did session)
                                                       :limit limit}
                                                cursor (assoc :cursor cursor))
                               :content-type :json
                               :as :json})
             :body
             (utils/validate-with-throw post-res-schema))
         posts (map :post (:feed body))
         new-cursor (:cursor body)
         new-posts (take-while #(not= (:cid %) id) posts)
         new-and-prev-posts (concat new-posts prev-posts)]
     (cond
       ;; end of posts
       (not= (count posts) limit) new-and-prev-posts
       ;; found post
       (some #(= id (:cid %)) posts) new-and-prev-posts
       ;; recur
       :else (recur session id new-cursor new-and-prev-posts)))))

(defn extract-tags [post]
  (let [facets (get (post :record) :facets [])
        features (flatten (map :features facets))
        tag-features (filter #(= (:$type %) "app.bsky.richtext.facet#tag") features)
        tags (map :tag tag-features)]
    tags))

(defn extract-images [post]
  (let [images (get-in post [:embed :images] [])]
    (map #(vector (:fullsize %) (:alt %)) images)))

(defn transform-post [post]
  (hash-map :source :blue_sky
            :fullPost post
            :remoteId (:cid post)
            :isTech (micro-blog.is-tech/is-tech? (:record post))
            :authorId (get-in post [:author :handle])
            :tags (extract-tags post)
            :images (extract-images post)
            :posted (get-in post [:record :createdAt])))

(defn save-post [post]
  (tel/log! {:level :info :data {:postId (:remoteId post)}} "Saving post")
  (pb/save-post post))

(defn run []
  (tel/log! :info "Running blue sky fetcher")
  (let [session (create-session)
        last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky)
        new-posts (reverse (get-posts-until-id session last-saved-id))]
    (doseq [post new-posts]
      (-> post transform-post save-post))))

1	(ns micro-blog.blue-sky
2	(:require
3	[clj-http.client :as http-client]
4	[micro-blog.pocket-base :as pb]
5	[micro-blog.utils :as utils]
6	[micro-blog.is-tech]
7	[taoensso.telemere :as tel]
8	[micro-blog.config :refer [config]]))
9
10	(defn create-session []
11	(let [identifier (@config :blue-sky-username)
12	api-key (@config :blue-sky-api-key)
13	body {:identifier identifier :password api-key}
14	url (str (@config :blue-sky-host) "/com.atproto.server.createSession")
15	res-schema [:map
16	[:did string?]
17	[:accessJwt string?]]]
18	(-> (http-client/post url
19	{:form-params body
20	:content-type :json
21	:as :json})
22	:body
23	(utils/validate-with-throw res-schema)
24	(#(assoc % :access-jwt (:accessJwt %)))
25	(select-keys [:did :access-jwt]))))
26
27	(def post-res-schema [:map
28	[:cursor [:maybe :string]]
29	[:feed [:vector
30	[:map [:post [:map
31	[:cid :string]
32	[:author [:map
33	[:handle :string]]]
34
35	[:embed {:optional true}
36	[:map [:images {:optional true} [:vector [:map
37	[:fullsize :string]
38	[:alt :string]]]]]]
39	[:record [:map
40	[:facets {:optional true} [:vector [:map
41	[:features [:vector [:map
42	[:$type :string]
43	[:tag {:optional true} [:maybe :string]]]]]]]]
44	[:createdAt :string]]]]]]]]])
45
46	(defn get-posts-until-id
47	([session id] (get-posts-until-id session id nil []))
48	([session id cursor prev-posts]
49	(tel/log! {:level :info :data {:postId (:remoteId id)}} "Getting posts until id")
50	(let [limit 5
51	body
52	(-> (http-client/get (str (@config :blue-sky-host) "/app.bsky.feed.getAuthorFeed")
53	{:headers {"Authorization" (str "Bearer " (session :access-jwt))}
54	:query-params (cond-> {:actor (:did session)
55	:limit limit}
56	cursor (assoc :cursor cursor))
57	:content-type :json
58	:as :json})
59	:body
60	(utils/validate-with-throw post-res-schema))
61	posts (map :post (:feed body))
62	new-cursor (:cursor body)
63	new-posts (take-while #(not= (:cid %) id) posts)
64	new-and-prev-posts (concat new-posts prev-posts)]
65	(cond
66	;; end of posts
67	(not= (count posts) limit) new-and-prev-posts
68	;; found post
69	(some #(= id (:cid %)) posts) new-and-prev-posts
70	;; recur
71	:else (recur session id new-cursor new-and-prev-posts)))))
72
73	(defn extract-tags [post]
74	(let [facets (get (post :record) :facets [])
75	features (flatten (map :features facets))
76	tag-features (filter #(= (:$type %) "app.bsky.richtext.facet#tag") features)
77	tags (map :tag tag-features)]
78	tags))
79
80	(defn extract-images [post]
81	(let [images (get-in post [:embed :images] [])]
82	(map #(vector (:fullsize %) (:alt %)) images)))
83
84	(defn transform-post [post]
85	(hash-map :source :blue_sky
86	:fullPost post
87	:remoteId (:cid post)
88	:isTech (micro-blog.is-tech/is-tech? (:record post))
89	:authorId (get-in post [:author :handle])
90	:tags (extract-tags post)
91	:images (extract-images post)
92	:posted (get-in post [:record :createdAt])))
93
94	(defn save-post [post]
95	(tel/log! {:level :info :data {:postId (:remoteId post)}} "Saving post")
96	(pb/save-post post))
97
98	(defn run []
99	(tel/log! :info "Running blue sky fetcher")
100	(let [session (create-session)
101	last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky)
102	new-posts (reverse (get-posts-until-id session last-saved-id))]
103	(doseq [post new-posts]
104	(-> post transform-post save-post))))
105

is_tech.clj · 878 B · Clojure Raw

1	(ns micro-blog.is-tech
2	(:require
3	[micro-blog.config :refer [config]]
4	[taoensso.telemere :as tel]
5	[clj-http.client :as client]))
6
7	(defn is-tech? [post-text]
8	(let [url (str (:mistral-host @config) "/v1/conversations")
9	headers {"Content-Type" "application/json"
10	"Accept" "application/json"
11	"Authorization" (str "Bearer " (@config :mistral-api-key))}
12	body {:inputs post-text
13	:stream false
14	:agent_id (@config :mistral-agent-id)}]
15	(tel/log! {:level :info :data {:url url :agent_id (:agent_id body)}} "making request to mistral agent")
16	(->
17	(client/post url {:headers headers
18	:form-params body
19	:content-type :json
20	:as :json})
21	:body
22	:outputs
23	first
24	:content
25	(#(if (= "1" %) true false)))))
26