diff --git a/Gemfile b/Gemfile index 9332728..6f831a4 100644 --- a/Gemfile +++ b/Gemfile @@ -7,6 +7,7 @@ ruby '3.1.4' gem 'awesome_print' gem 'app_version', github: "afaraldo/app_version", branch: "master" +gem 'semantic' gem 'gammo' gem 'pry' diff --git a/Gemfile.lock b/Gemfile.lock index 3baa609..bc82eac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -273,6 +273,7 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) + semantic (1.6.1) sprockets (4.2.0) concurrent-ruby (~> 1.0) rack (>= 2.2.4, < 4) @@ -339,6 +340,7 @@ DEPENDENCIES rails (~> 7.0.5) rspec-rails (~> 6.0.3) selenium-webdriver + semantic sprockets-rails stimulus-rails thor diff --git a/Procfile b/Procfile index 780be83..905bd74 100644 --- a/Procfile +++ b/Procfile @@ -1,7 +1,9 @@ sample: bundle exec ruby ./cli sample # projer: bundle exec ruby ./cli proj leafer: bundle exec ruby ./cli leaf -# leafer2: bundle exec ruby ./cli leaf2 -# leafer3: bundle exec ruby ./cli leaf3 -# leafer4: bundle exec ruby ./cli leaf4 measure: bundle exec ruby ./cli measure + +pkvsample: bundle exec ruby ./cli pkvsample +# packageversioner: bundle exec ruby ./cli packageversion +# versionleafer: bundle exec ruby ./cli versionleaf +# measver: bundle exec ruby ./cli measureversion diff --git a/app/models/version.rb b/app/models/version.rb new file mode 100644 index 0000000..e27b804 --- /dev/null +++ b/app/models/version.rb @@ -0,0 +1,3 @@ +class Version < ApplicationRecord + belongs_to :package +end diff --git a/app/models/version_measurement.rb b/app/models/version_measurement.rb new file mode 100644 index 0000000..0fb187f --- /dev/null +++ b/app/models/version_measurement.rb @@ -0,0 +1,4 @@ +class VersionMeasurement < ApplicationRecord + belongs_to :package + belongs_to :version +end diff --git a/config/version.yml b/config/version.yml index c65206c..b92d437 100644 --- a/config/version.yml +++ b/config/version.yml @@ -5,13 +5,13 @@ major: 0 minor: 1 -patch: 2 +patch: 3 # meta: rc.1 # milestone: 4 -build: 244 +build: 260 committer: Kingdon Barrett -build_date: 2023-06-27 +build_date: 2023-07-02 diff --git a/db/migrate/20230702152802_create_versions.rb b/db/migrate/20230702152802_create_versions.rb new file mode 100644 index 0000000..80d7963 --- /dev/null +++ b/db/migrate/20230702152802_create_versions.rb @@ -0,0 +1,11 @@ +class CreateVersions < ActiveRecord::Migration[7.0] + def change + create_table :versions do |t| + t.references :package, null: false, foreign_key: true + t.string :version + t.integer :download_count + + t.timestamps + end + end +end diff --git a/db/migrate/20230702152809_create_version_measurements.rb b/db/migrate/20230702152809_create_version_measurements.rb new file mode 100644 index 0000000..e4b5ca9 --- /dev/null +++ b/db/migrate/20230702152809_create_version_measurements.rb @@ -0,0 +1,12 @@ +class CreateVersionMeasurements < ActiveRecord::Migration[7.0] + def change + create_table :version_measurements do |t| + t.references :package, null: false, foreign_key: true + t.references :version, null: false, foreign_key: true + t.integer :count + t.datetime :measured_at + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 3a66948..4f49850 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_06_01_141806) do +ActiveRecord::Schema[7.0].define(version: 2023_07_02_152809) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -47,7 +47,30 @@ t.index ["github_org_id"], name: "index_repositories_on_github_org_id" end + create_table "version_measurements", force: :cascade do |t| + t.bigint "package_id", null: false + t.bigint "version_id", null: false + t.integer "count" + t.datetime "measured_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["package_id"], name: "index_version_measurements_on_package_id" + t.index ["version_id"], name: "index_version_measurements_on_version_id" + end + + create_table "versions", force: :cascade do |t| + t.bigint "package_id", null: false + t.string "version" + t.integer "download_count" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["package_id"], name: "index_versions_on_package_id" + end + add_foreign_key "measurements", "packages" add_foreign_key "packages", "repositories" add_foreign_key "repositories", "github_orgs" + add_foreign_key "version_measurements", "packages" + add_foreign_key "version_measurements", "versions" + add_foreign_key "versions", "packages" end diff --git a/deploy/bases/crds/crds.yml b/deploy/bases/crds/crds.yml index a8e9974..a5ff376 100644 --- a/deploy/bases/crds/crds.yml +++ b/deploy/bases/crds/crds.yml @@ -154,6 +154,7 @@ spec: type: "string" minimum: 1 conditions: + description: Conditions holds the conditions for the Leaf items: properties: lastTransitionTime: @@ -202,7 +203,114 @@ spec: type: string observedGeneration: description: ObservedGeneration is the last observed generation of - the Project object. + the Leaf object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: "apiextensions.k8s.io/v1" +kind: "CustomResourceDefinition" +metadata: + name: "packageversions.example.com" +spec: + group: "example.com" + names: + plural: "packageversions" + singular: "packageversion" + kind: "PackageVersion" + shortNames: + - pkv + scope: "Namespaced" + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: "v1alpha1" + schema: + openAPIV3Schema: + required: ["spec"] + properties: + spec: + required: ["projectName", "packageName"] + properties: + projectName: + type: "string" + minimum: 1 + packageName: + type: "string" + minimum: 1 + type: object + status: + properties: + count: + type: "string" + minimum: 1 + lastUpdate: + type: "string" + minimum: 1 + conditions: + description: Conditions holds the conditions for the PackageVersion + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + lastHandledReconcileAt: + description: LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value can + be detected. + type: string + observedGeneration: + description: ObservedGeneration is the last observed generation of + the PackageVersion object. format: int64 type: integer type: object @@ -211,3 +319,113 @@ spec: storage: true subresources: status: {} +# --- +# apiVersion: "apiextensions.k8s.io/v1" +# kind: "CustomResourceDefinition" +# metadata: +# name: "versionleaves.example.com" +# spec: +# group: "example.com" +# names: +# plural: "versionleaves" +# singular: "versionleaf" +# kind: "VersionLeaf" +# shortNames: +# - vl +# scope: "Namespaced" +# versions: +# - additionalPrinterColumns: +# - jsonPath: .metadata.creationTimestamp +# name: Age +# type: date +# - jsonPath: .status.conditions[?(@.type=="Ready")].status +# name: Ready +# type: string +# - jsonPath: .status.conditions[?(@.type=="Ready")].message +# name: Status +# type: string +# name: "v1alpha1" +# schema: +# openAPIV3Schema: +# required: ["spec"] +# properties: +# spec: +# required: ["projectName", "packageName", "version"] +# properties: +# projectName: +# type: "string" +# minimum: 1 +# packageName: +# type: "string" +# minimum: 1 +# version: +# type: "string" +# minimum: 1 +# type: object +# status: +# properties: +# count: +# type: "string" +# minimum: 1 +# lastUpdate: +# type: "string" +# minimum: 1 +# conditions: +# description: Conditions holds the conditions for the VersionLeaf +# items: +# properties: +# lastTransitionTime: +# format: date-time +# type: string +# message: +# description: message is a human readable message indicating +# details about the transition. This may be an empty string. +# maxLength: 32768 +# type: string +# observedGeneration: +# description: observedGeneration represents the .metadata.generation +# that the condition was set based upon. +# format: int64 +# minimum: 0 +# type: integer +# reason: +# description: reason contains a programmatic identifier +# maxLength: 1024 +# minLength: 1 +# pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ +# type: string +# status: +# description: status of the condition, one of True, False, Unknown. +# enum: +# - "True" +# - "False" +# - Unknown +# type: string +# type: +# maxLength: 316 +# pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ +# type: string +# required: +# - lastTransitionTime +# - message +# - reason +# - status +# - type +# type: object +# type: array +# lastHandledReconcileAt: +# description: LastHandledReconcileAt holds the value of the most recent +# reconcile request value, so a change of the annotation value can +# be detected. +# type: string +# observedGeneration: +# description: ObservedGeneration is the last observed generation of +# the VersionLeaf object. +# format: int64 +# type: integer +# type: object +# type: object +# served: true +# storage: true +# subresources: +# status: {} diff --git a/deploy/bases/cron/cronjob.yaml b/deploy/bases/cron/cronjob.yaml index c3ecb31..a98bd55 100644 --- a/deploy/bases/cron/cronjob.yaml +++ b/deploy/bases/cron/cronjob.yaml @@ -13,7 +13,7 @@ spec: containers: - name: stathcr image: ghcr.io/kingdonb/stats-tracker-ghcr:canary - imagePullPolicy: Always + imagePullPolicy: IfNotPresent #args: # - "-v=3" # - --privateKeyPath=/etc/secret-volume/privatekey.pem diff --git a/deploy/bases/test/pkvsample.yml b/deploy/bases/test/pkvsample.yml new file mode 100644 index 0000000..682399e --- /dev/null +++ b/deploy/bases/test/pkvsample.yml @@ -0,0 +1,66 @@ +--- +kind: PackageVersion +apiVersion: example.com/v1alpha1 +metadata: + name: "kustomize-controller" +spec: + projectName: "fluxcd" + packageName: "kustomize-controller" +--- +kind: PackageVersion +apiVersion: example.com/v1alpha1 +metadata: + name: "source-controller" +spec: + projectName: "fluxcd" + packageName: "source-controller" +--- +kind: PackageVersion +apiVersion: example.com/v1alpha1 +metadata: + name: "notification-controller" +spec: + projectName: "fluxcd" + packageName: "notification-controller" +--- +kind: PackageVersion +apiVersion: example.com/v1alpha1 +metadata: + name: "image-automation-controller" +spec: + projectName: "fluxcd" + packageName: "image-automation-controller" +--- +kind: PackageVersion +apiVersion: example.com/v1alpha1 +metadata: + name: "helm-controller" +spec: + projectName: "fluxcd" + packageName: "helm-controller" +--- +kind: PackageVersion +apiVersion: example.com/v1alpha1 +metadata: + name: "image-reflector-controller" +spec: + projectName: "fluxcd" + packageName: "image-reflector-controller" +--- +kind: PackageVersion +apiVersion: example.com/v1alpha1 +metadata: + name: "flagger" +spec: + projectName: "fluxcd" + packageName: "flagger" +--- +kind: VersionLeaf +apiVersion: example.com/v1alpha1 +metadata: + name: "kustomize-controller" +spec: + repoName: "kustomize-controller" + projectName: "fluxcd" + packageName: "kustomize-controller" + version: "v1.0.0-rc.4" diff --git a/deploy/overlays/dev/kustomization.yaml b/deploy/overlays/dev/kustomization.yaml index cae1401..b8aef63 100644 --- a/deploy/overlays/dev/kustomization.yaml +++ b/deploy/overlays/dev/kustomization.yaml @@ -9,6 +9,10 @@ resources: - namespace.yaml patches: - patch: |- + - op: replace + path: /spec/schedule + value: "*/4 * * * *" + # Dev schedule runs every 4m - op: replace path: /spec/jobTemplate/spec/template/spec/containers/0/env/1 value: @@ -26,3 +30,14 @@ patches: target: kind: Deployment name: stats-viewer +- patch: | + - op: replace + path: /spec/jobTemplate/spec/template/spec/containers/0/imagePullPolicy + value: Always + target: + kind: CronJob + name: stats-tracker-ghcr +images: +- name: ghcr.io/kingdonb/stats-tracker-ghcr + newName: ghcr.io/kingdonb/stats-tracker-ghcr + newTag: canary diff --git a/deploy/overlays/production/kustomization.yaml b/deploy/overlays/production/kustomization.yaml index 9d9f3fa..e09050f 100644 --- a/deploy/overlays/production/kustomization.yaml +++ b/deploy/overlays/production/kustomization.yaml @@ -10,12 +10,5 @@ resources: images: - name: ghcr.io/kingdonb/stats-tracker-ghcr newName: ghcr.io/kingdonb/stats-tracker-ghcr - newTag: 0.1.2 -patches: -- patch: | - - op: replace - path: /spec/jobTemplate/spec/template/spec/containers/0/imagePullPolicy - value: IfNotPresent - target: - kind: CronJob - name: stats-tracker-ghcr + newTag: 0.1.3 +patches: [] diff --git a/lib/cli.rb b/lib/cli.rb index 8c47a4e..7f6968e 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -7,6 +7,10 @@ require './lib/leaf_reconciler' require './lib/sample' require './app/models/measurement' +require './lib/package_version_reconciler' +# require './lib/version_leaf_reconciler' +require './lib/pkv_sample' +require './app/models/version_measurement' class MyCLI < Thor @@ -38,4 +42,33 @@ def measure() # Fiber.set_scheduler(FiberScheduler.new) Measurement.call end + + desc "pkvsample PKG", "Create a PackageVersion for the Kustomize Controller (or PKG) and Reconcile PackageVersions" + def pkvsample(name: "fluxcd", pkvname: "kustomize-controller") + packageversioner = PackageVersion::Operator.new + PkvSample.ensure() + packageversioner.run + end + + desc "packageversion", "Reconcile the packageversions" + def packageversion() + # Fiber.set_scheduler(FiberScheduler.new) + packageversioner = PackageVersion::Operator.new + + packageversioner.run + end + + # desc "versionleaf", "Reconcile the versionleaves" + # def versionleaf() + # Fiber.set_scheduler(FiberScheduler.new) + # versionleafer = VersionLeaf::Operator.new + + # versionleafer.run + # end + + # desc "measure", "Do the measurement (PackageVersion/Leaves)" + # def measver() + # # Fiber.set_scheduler(FiberScheduler.new) + # VersionMeasurement.call + # end end diff --git a/lib/package_version_reconciler.rb b/lib/package_version_reconciler.rb new file mode 100644 index 0000000..1198464 --- /dev/null +++ b/lib/package_version_reconciler.rb @@ -0,0 +1,128 @@ +require 'kubernetes-operator' +require 'open-uri' +require 'nokogiri' +require 'yaml' +require 'semantic' + +# basedir = File.expand_path('../app/models', __FILE__) +# Dir["#{basedir}/*.rb"].each do |path| +# name = "#{File.basename(path, '.rb')}" +# autoload name.classify.to_sym, "#{basedir}/#{name}" +# end + +require 'active_record' +require './app/models/application_record' +# Bundler.require(*Rails.groups) +require 'pg' +require 'dotenv' + +require './app/models/package' +require './app/models/version' +require './app/models/version_measurement' +require './lib/ar_base_connection' + +module PackageVersion + class Operator + require './lib/operator_utility' + def initialize + end + + def run + crdVersion = "v1alpha1" + crdPlural = "packageversions" + @api = AR::BaseConnection. + new(version: crdVersion, plural: crdPlural, poolSize: 1) + + init_k8s_only + @opi.run + end + + def upsert(obj) + projectName = obj["spec"]["projectName"] + packageName = obj["spec"]["packageName"] + @logger.info("create new packageversion {projectName: #{projectName}, packageName: #{packageName}}") + + create_new_leaves(obj) + time_t0 = DateTime.now.in_time_zone.to_time + + k8s = @opi.instance_variable_get("@k8sclient") + + pack = nil + loop do + pack = Package.find_by(name: packageName) + break if pack.present? + sleep 2 + end + + overall_count = 0 + VersionMeasurement.transaction do |t| + @ts.each do |t| + v = t[0] + count = t[1][0] + + begin + version = Semantic::Version.new(v) + vers = Version.find_or_create_by(package: pack, version: version.to_s) + measure = VersionMeasurement.new( + measured_at: time_t0, + count: count, + package: pack, + version: vers + ) + measure.save! + + overall_count = overall_count + 1 + + rescue ArgumentError + # filter as it did not parse as semver + end + end + end + + @logger.info("reached the end for packageversion {projectName: #{projectName}, packageName: #{packageName}}") + time_t1 = DateTime.now.in_time_zone.to_time + + {:status => { + :count => overall_count.to_s, + :lastUpdate => time_t1, + # :conditions => + }} + end + + def delete(obj) + end + + def create_new_leaves(obj) + # name = obj["metadata"]["name"] + project = obj["spec"]["projectName"] + package = obj["spec"]["packageName"] + + client = Proc.new do |url| + URI.open(url) + end + + url = "https://github.com/#{project}/#{package}/pkgs/container/#{package}/versions?filters%5Bversion_type%5D=tagged" + c = client.call(url) + html = c.read + h = Nokogiri::HTML(html) + + d = h.css("a.Label") + @ts = {} + + d.map{|t| + v = t.text + # Ignore these signatures that do not look like semver + if v[0] == "v" + version = v[1..] + ppp = t.parent.parent.parent + counter = ppp.css('div span.color-fg-muted.overflow-hidden.f6.mr-3') + count = counter.text.strip.gsub(',','').to_i + + @ts[version] = [count] + end + } + + # create one Leaf for each t in ts + end + end +end diff --git a/lib/pkv_sample.rb b/lib/pkv_sample.rb new file mode 100644 index 0000000..15f8dd8 --- /dev/null +++ b/lib/pkv_sample.rb @@ -0,0 +1,48 @@ +require 'yaml' +require 'ap' +require './lib/ar_base_connection' + +class PkvSample + def self.ensure() + crdVersion = "v1alpha1" + crdPlural = "packageversions" + api = AR::BaseConnection. + new(version: crdVersion, plural: crdPlural, poolSize: 1) + operator = api[:opi] + k8s = operator. + instance_variable_get("@k8sclient") + docs = YAML.load_file('./deploy/bases/test/pkvsample.yml') + + if docs.class == Hash + docs = [docs] + end + + docs.each do |pkv| + if pkv["kind"] == "PackageVersion" + name = pkv["metadata"]["name"] + projectName = pkv["spec"]["projectName"] + packageName = pkv["spec"]["packageName"] + begin + p = k8s.get_package_version(name, 'default') + if p.respond_to?(:kind) + next # pkv is already present on the cluster, + end + rescue Kubeclient::ResourceNotFoundError => e + # this is the signal to proceed, create the project + end + + k8s.create_package_version(Kubeclient::Resource.new({ + metadata: { + name: name, namespace: 'default' + }, + spec: { + projectName: projectName, + packageName: packageName + } + })) + else + raise StandardError, "Sample yaml was not a PackageVersion as expected" + end + end + end +end diff --git a/lib/templates/version.yml.erb b/lib/templates/version.yml.erb index 30da3ac..dc52a4f 100644 --- a/lib/templates/version.yml.erb +++ b/lib/templates/version.yml.erb @@ -5,7 +5,7 @@ major: 0 minor: 1 -patch: 2 +patch: 3 # meta: rc.1 # milestone: 4 build: <%= `git rev-list HEAD|wc -l`.strip %> diff --git a/spec/factories/version_measurements.rb b/spec/factories/version_measurements.rb new file mode 100644 index 0000000..13e773b --- /dev/null +++ b/spec/factories/version_measurements.rb @@ -0,0 +1,8 @@ +FactoryBot.define do + factory :version_measurement do + package { nil } + version { nil } + count { 1 } + measured_at { "2023-07-02 11:28:09" } + end +end diff --git a/spec/factories/versions.rb b/spec/factories/versions.rb new file mode 100644 index 0000000..dff1946 --- /dev/null +++ b/spec/factories/versions.rb @@ -0,0 +1,7 @@ +FactoryBot.define do + factory :version do + package { nil } + version { "MyString" } + download_count { 1 } + end +end diff --git a/spec/models/version_measurement_spec.rb b/spec/models/version_measurement_spec.rb new file mode 100644 index 0000000..f02e426 --- /dev/null +++ b/spec/models/version_measurement_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe VersionMeasurement, type: :model do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/models/version_spec.rb b/spec/models/version_spec.rb new file mode 100644 index 0000000..6ca5621 --- /dev/null +++ b/spec/models/version_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe Version, type: :model do + pending "add some examples to (or delete) #{__FILE__}" +end