Skip to content

Commit

Permalink
Merge pull request #18 from clj-codes/feat/adds-more-libs-n-fixes
Browse files Browse the repository at this point in the history 
feat: add more libs to the database
  • Loading branch information
@rafaeldelboni
rafaeldelboni authored Mar 23, 2024
2 parents 44c2144 + 005a2e6 commit cd52d6c
Show file tree
Hide file tree
Showing 9 changed files with 316 additions and 44 deletions.
47 changes: 45 additions & 2 deletions dev/playground.clj
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
(ns dev.playground
(:require [clj-http.client :as http]
[clojure.java.io :as io]
[codes.clj.docs.extractor.config :as config]
[codes.clj.docs.extractor.core :as core]
[codes.clj.docs.extractor.datalevin :as datalevin]
[datalevin.core :as d]
Expand All @@ -10,7 +11,8 @@
(:import [java.io File]))

(defn get-url [git-url]
(-> (http/get git-url {:as :json})
(-> (http/get git-url {;:headers {"Authorization" "Bearer token"}
:as :json})
:body))

(defn download-unzip [dir url]
Expand All @@ -31,7 +33,34 @@
(io/copy stream saveFile))))
(recur (.getNextEntry stream))))))

(defn get-github-data [owner-repository]
(let [tags (get-url
(str "https://api.github.com/repos/"
owner-repository
"/tags"))
latest (first tags)
latest-hash (get-url
(str "https://api.github.com/repos/"
owner-repository
"/git/ref/tags/"
(:name latest)))]
(merge latest latest-hash)))

(defn get-git-deps-info [owner-repository]
(let [{:keys [name object]} (-> owner-repository
get-github-data
(select-keys [:name :object]))]
{:project/name ""
:git/url (str "https://github.com/" owner-repository)
:git/tag name
:git/sha (:sha object)}))

(comment
; getting data from github
(mapv get-git-deps-info
["dakrone/clj-http"
"dakrone/cheshire"])

; reset database & download unzip from releases
(let [dir "target/docs-db"]
(println "deleting")
Expand Down Expand Up @@ -134,13 +163,17 @@
(d/close conn)
result)

; count deps in config
(count (:deps (config/read! "resources/config.edn")))

; count by project
(let [conn (d/get-conn "target/docs-db" datalevin/db-schemas)
db (d/db conn)
result (doall (d/q '[:find ?pn ?ps (count ?d)
result (doall (d/q '[:find ?pn ?pg (count ?d)
:in $
:where
[?p :project/id]
[?p :project/group ?pg]
[?p :project/name ?pn]
[?p :project/sha ?ps]
[?n :namespace/project ?p]
Expand All @@ -162,6 +195,16 @@
(d/close conn)
result)

(let [conn (d/get-conn "target/docs-db" datalevin/db-schemas)
db (d/db conn)
result (doall (d/q '[:find ?e ?a
:in $
:where
[?e :definition/deprecated ?a]]
db))]
(d/close conn)
result)

; tests with fulltext and analyzer
(let [query-analyzer (su/create-analyzer
{:tokenizer (datalevin/merge-tokenizers
Expand Down
162 changes: 161 additions & 1 deletion resources/config.edn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,164 @@
org.clojure/data.json
{:git/url "https://github.com/clojure/data.json"
:git/tag "v2.5.0"
:git/sha "457b88e27d7695b38f8b3d9c737e2efb8f2b476d"}}}
:git/sha "457b88e27d7695b38f8b3d9c737e2efb8f2b476d"}

nubank/matcher-combinators
{:git/url "https://github.com/nubank/matcher-combinators"
:git/tag "3.9.1"
:git/sha "beae388ac30d4079a68619aba36ff7e152b20e42"}
nubank/state-flow
{:deps/manifest :deps
:git/url "https://github.com/nubank/state-flow"
:git/tag "5.14.5"
:git/sha "e86f5136f470c3816e807888c510190b6496f59f"}
nubank/mockfn
{:deps/manifest :deps
:git/url "https://github.com/nubank/mockfn"
:git/tag "0.7.0"
:git/sha "7e51406edd237ba51687bfef4e66ee521c603b5f"}

metosin/malli
{:deps/manifest :deps
:git/url "https://github.com/metosin/malli"
:git/tag "0.15.0"
:git/sha "8f240d75131a77056303f3a144ad49889d08139f"}
metosin/reitit
{:deps/manifest :deps
:project/source-paths ["modules/reitit/src"
"modules/reitit-core/src"
"modules/reitit-dev/src"
"modules/reitit-ring/src"
"modules/reitit-http/src"
"modules/reitit-middleware/src"
"modules/reitit-openapi/src"
"modules/reitit-interceptors/src"
"modules/reitit-malli/src"
"modules/reitit-spec/src"
"modules/reitit-schema/src"
"modules/reitit-swagger/src"
"modules/reitit-swagger-ui/src"
"modules/reitit-frontend/src"
"modules/reitit-sieppari/src"
"modules/reitit-pedestal/src"]
:git/url "https://github.com/metosin/reitit"
:git/tag "0.7.0-alpha7"
:git/sha "15e0c95cb6e8ab1f653204f910dfb1851c0b0663"}
metosin/jsonista
{:git/url "https://github.com/metosin/jsonista"
:git/tag "0.3.8"
:git/sha "8aca9b09ba1964245438dfc104ea26f4ae9c4d21"}

bidi/bidi
{:project/group "juxt"
:deps/manifest :deps
:git/url "https://github.com/juxt/bidi"
:git/tag "2.1.6"
:git/sha "06385ef65a54bb8e234da79e67d629403c224797"}
aero/aero
{:project/group "juxt"
:git/url "https://github.com/juxt/aero"
:git/tag "1.1.6"
:git/sha "c47a10fa5f6a52084d04769af06d5e04d6603e13"}
tick/tick
{:project/group "juxt"
:git/url "https://github.com/juxt/tick"
:git/tag "0.7.5"
:git/sha "383fed95e906f5b877480b5cf9e0951a8fee4908"}

aleph/aleph
{:project/group "org.clj-commons"
:git/url "https://github.com/clj-commons/aleph"
:git/tag "0.7.1"
:git/sha "f4f6ab07833bd9054c53f5a411d66a4f39ab3715"}
manifold/manifold
{:deps/manifest :deps
:project/group "org.clj-commons"
:git/url "https://github.com/clj-commons/manifold"
:git/tag "0.4.2"
:git/sha "3188eec251dce9508a868cac1f860505ad3b066f"}
org.clj-commons/hickory
{:git/url "https://github.com/clj-commons/hickory"
:git/tag "Release-0.7.4"
:git/sha "139484a9066ccac5df31303b5cfc53d0379ffc5d"}

babashka/sci
{:git/url "https://github.com/babashka/sci"
:git/tag "v0.8.41"
:git/sha "e9fca84a40d975c276717088136bf4c996f9c506"}
babashka/fs
{:git/url "https://github.com/babashka/fs"
:git/tag "v0.5.20"
:git/sha "40fcceb22b9728fee1c2ce0e58e390e5fe23f2cd"}
babashka/process
{:git/url "https://github.com/babashka/process"
:git/tag "v0.5.22"
:git/sha "c8a5d8ce466dd57ac9680fe526846c37fd2a06fe"}

lambdaisland/kaocha
{:git/url "https://github.com/lambdaisland/kaocha"
:git/tag "v1.87.1366"
:git/sha "17970292cc99710818f01ac63b61b62dbd3afa23"}
lambdaisland/regal
{:git/url "https://github.com/lambdaisland/regal"
:git/tag "v0.0.143"
:git/sha "c99d0b1d66bea9f9b4b772a095c9181deda89580"}
lambdaisland/uri
{:git/url "https://github.com/lambdaisland/uri"
:git/tag "v1.19.155"
:git/sha "1b0a2d0d6845af5dd45d3f8685d1db9a20ca56ef"}

compojure/compojure
{:deps/manifest :deps
:project/group "weavejester"
:git/url "https://github.com/weavejester/compojure"
:git/tag "1.7.1"
:git/sha "55268be19da2460b08a5cc4e8ac43bdb3baf55b9"}
hiccup/hiccup
{:deps/manifest :deps
:project/group "weavejester"
:git/url "https://github.com/weavejester/hiccup"
:git/tag "2.0.0-alpha2"
:git/sha "56ecd8f7bf70614b69fa2fba8e99a169f6efd5cc"}
integrant/integrant
{:project/group "weavejester"
:git/url "https://github.com/weavejester/integrant"
:git/tag "0.9.0-alpha4"
:git/sha "e74a0924e2bf26d4b02649319c04b79c64823a89"}

com.taoensso/sente
{:deps/manifest :deps
:git/url "https://github.com/taoensso/sente"
:git/tag "v1.19.2"
:git/sha "71f993b59d4128376108e2e036bc25ad70723a37"}
com.taoensso/timbre
{:deps/manifest :deps
:git/url "https://github.com/taoensso/timbre"
:git/tag "v6.5.0"
:git/sha "b72cc65290cf2e5136cfa0a3dd449eb806e31ff5"}
com.taoensso/carmine
{:deps/manifest :deps
:git/url "https://github.com/taoensso/carmine"
:git/tag "v3.3.2"
:git/sha "73b846204c6a9ca2168aa3dbaa1d3f2b151c7e66"}

clj-http/clj-http
{:deps/manifest :deps
:project/group "dakrone"
:git/url "https://github.com/dakrone/clj-http"
:git/tag "3.12.3"
:git/sha "99f00af3273aec1fb7bf51e804ecddb2a34a50f2"}
cheshire/cheshire
{:deps/manifest :deps
:project/group "dakrone"
:git/url "https://github.com/dakrone/cheshire"
:git/tag "5.12.0"
:git/sha "08b1d1c8bc8985a9faeea62fca7d8c8f3696721d"}

lilactown/helix
{:git/url "https://github.com/lilactown/helix",
:git/sha "35127b79405e5dff6d9b74dfc674280eb93fab6d"}
town.lilac/flex
{:project/group "lilactown"
:git/url "https://github.com/lilactown/flex",
:git/sha "afa6491ace9015577ffa9245086f945825a93633"}}}
16 changes: 7 additions & 9 deletions src/codes/clj/docs/extractor/adapters.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
[analysis]
(mapv
(fn [{:keys [project]}]
(let [[group artifact] (-> project :project-name (str/split #"/"))]
(let [{:keys [group artifact]} project]
(assoc-some {:project/id (:project-name project)
:project/name (:project-name project)}
:project/group group
Expand All @@ -73,8 +73,7 @@
(reduce
(fn [accum {:keys [project namespaces]}]
(into accum
(let [{:git/keys [url tag] :deps/keys [root]} project
[group artifact] (-> project :project-name (str/split #"/"))]
(let [{:keys [group artifact] :git/keys [url tag sha] :deps/keys [root]} project]
(->> namespaces
group-multi-langs
(mapv (fn [{:keys [end-row meta name-end-col name-end-row name-row deprecated added
Expand All @@ -91,11 +90,11 @@
:namespace/name-end-col name-end-col
:namespace/name-end-row name-end-row
:namespace/name-row name-row
:namespace/deprecated deprecated
:namespace/deprecated (some-> deprecated str)
:namespace/added added
:namespace/author author
:namespace/filename trim-filename
:namespace/git-source (str url "/blob/" tag trim-filename "#L" row)
:namespace/git-source (str url "/blob/" (or tag sha) trim-filename "#L" row)
:namespace/col col
:namespace/name-col name-col
:namespace/end-col end-col
Expand All @@ -115,8 +114,7 @@
(reduce
(fn [accum {:keys [project definitions]}]
(into accum
(let [{:git/keys [url tag] :deps/keys [root]} project
[group artifact] (-> project :project-name (str/split #"/"))]
(let [{:keys [group artifact] :git/keys [url tag sha] :deps/keys [root]} project]
(->> definitions
group-multi-langs
(mapv (fn [{:keys [fixed-arities end-row meta name-end-col
Expand All @@ -141,11 +139,11 @@
:definition/name-end-row name-end-row
:definition/name-row name-row
:definition/added added
:definition/deprecated deprecated
:definition/deprecated (some-> deprecated str)
:definition/author author
:definition/filename trim-filename
:definition/git-source (when trim-filename
(str url "/blob/" tag trim-filename "#L" row))
(str url "/blob/" (or tag sha) trim-filename "#L" row))
:definition/col col
:definition/name-col name-col
:definition/end-col end-col
Expand Down
48 changes: 34 additions & 14 deletions src/codes/clj/docs/extractor/analysis.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,43 @@
(:require [clj-kondo.core :as kondo]
[clojure.edn :as edn]
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.tools.deps :as deps]))

(def ^:private mvn-repos
{"central" {:url "https://repo1.maven.org/maven2/"}
"clojars" {:url "https://repo.clojars.org/"}})

(defn build-paths [resolved-deps source-paths]
(if source-paths
(into (:paths resolved-deps)
(map #(str (:deps/root resolved-deps) "/" %) source-paths))
(:paths resolved-deps)))

(defn resolved-deps->project-meta
[resolved-deps project-name artifact group paths]
(assoc resolved-deps
:project-name project-name
:artifact artifact
:group group
:paths paths))

(defn download-project!
[project git]
(-> (deps/resolve-deps
{:deps {project git}
:mvn/repos {"central" {:url "https://repo1.maven.org/maven2/"},
"clojars" {:url "https://repo.clojars.org/"}}}
nil)
(get project)
(assoc :project-name (str project))))
(let [project-name (str project)
[group artifact] (str/split project-name #"/")
group-name (or (:project/group git) group)
resolved-deps (-> (deps/resolve-deps
{:deps {project git}
:mvn/repos mvn-repos}
nil)
(get project))
paths (build-paths resolved-deps (:project/source-paths git))]
(resolved-deps->project-meta resolved-deps
project-name
artifact
group-name
paths)))

(defn kondo-run!
[paths]
Expand All @@ -34,16 +60,10 @@

(defn extract-analysis!
[project-name project-config]

(let [{:keys [paths] :as project-meta} (download-project! project-name project-config)
{:keys [var-definitions namespace-definitions]} (kondo-run! paths)
extra-definitions (extract-extras! (-> project-config :extras :definitions))]
{:project project-meta
:namespaces namespace-definitions
:definitions (into var-definitions extra-definitions)}))

(defn extract!
[config]
(->> config
:deps
(mapv (fn [[project-name project-config]]
(extract-analysis! project-name project-config)))))
Loading

0 comments on commit cd52d6c

Please sign in to comment.