Update julia syncing code

Gemfile

@@ -44,7 +44,7 @@ gem 'google-protobuf'
 gem "xmlrpc"
 gem 'rexml'
 gem 'appsignal'
-# gem 'faraday-typhoeus'
+gem 'faraday-typhoeus'
 gem 'packageurl-ruby'
 gem 'sitemap_generator'
 gem 'dalli'

Gemfile.lock

@@ -143,6 +143,9 @@ GEM
       net-http (>= 0.5.0)
     faraday-retry (2.2.1)
       faraday (~> 2.0)
+    faraday-typhoeus (1.1.0)
+      faraday (~> 2.0)
+      typhoeus (~> 1.4)
     ffi (1.17.1)
     ffi (1.17.1-aarch64-linux-musl)
     ffi (1.17.1-arm-linux-gnu)
@@ -420,6 +423,7 @@ DEPENDENCIES
   faraday
   faraday-follow_redirects
   faraday-retry
+  faraday-typhoeus
   google-protobuf
   groupdate
   jbuilder
@@ -509,6 +513,7 @@ CHECKSUMS
   faraday-follow_redirects (0.3.0) sha256=d92d975635e2c7fe525dd494fcd4b9bb7f0a4a0ec0d5f4c15c729530fdb807f9
   faraday-net_http (3.4.0) sha256=a1f1e4cd6a2cf21599c8221595e27582d9936819977bbd4089a601f24c64e54a
   faraday-retry (2.2.1) sha256=4146fed14549c0580bf14591fca419a40717de0dd24f267a8ec2d9a728677608
+  faraday-typhoeus (1.1.0) sha256=24c6147c213818dde3ebc50ae47ab92f9a7e554903aa362707126f749c6890e7
   ffi (1.17.1) sha256=26f6b0dbd1101e6ffc09d3ca640b2a21840cc52731ad8a7ded9fb89e5fb0fc39
   ffi (1.17.1-aarch64-linux-musl) sha256=88b9d6ae905d21142df27c94bb300042c1aae41b67291885f600eaad16326b1d
   ffi (1.17.1-arm-linux-gnu) sha256=fe14f5ece94082f3b0e651a09008113281f2764e7ea95f522b64e2fe32e11504

app/models/ecosystem/julia.rb

@@ -8,11 +8,10 @@ def sync_in_batches?
     end
 
     def registry_url(package, version = nil)
-      "#{@registry_url}/ui/Packages/#{package.name}/#{package.metadata['slug']}/#{version}"
+      "#{@registry_url}/ui/Packages/General/#{package.name}/#{version}"
     end
 
     def check_status(package)
-      return "removed" if package.metadata['slug'].blank?
       url = check_status_url(package)
       connection = Faraday.new do |faraday|
         faraday.use Faraday::FollowRedirects::Middleware
@@ -24,7 +23,7 @@ def check_status(package)
     end
 
     def check_status_url(package)
-      "#{@registry_url}/docs/#{package['name']}/#{package.metadata['slug']}/pkg.json"
+      "#{@registry_url}/docs/General/#{package['name']}/stable/pkg.json"
     end
 
     def download_url(package, version = nil)
@@ -39,13 +38,9 @@ def download_url(package, version = nil)
       end
     end
 
-    def documentation_url(package, version = nil)
-      "https://docs.juliahub.com/#{package.name}/#{package.metadata['slug']}/#{version}"
-    end
-
     def install_command(package, version = nil)
       if version
-        "Pkg.add(Pkg.PackageSpec(;name=\"#{package.name}\", version=\"#{version}\"))"
+        "Pkg.add(\"#{package.name}@#{version}\")"
       else
         "Pkg.add(\"#{package.name}\")"
       end
@@ -77,23 +72,21 @@ def fetch_package_metadata(name)
 
     def map_package_metadata(package)
       return false unless package
-      return false unless package['metadata']['docslink']
-      slug = package['metadata']['docslink'].split('/')[2]
-      json = get_json("#{@registry_url}/docs/#{package['name']}/#{slug}/pkg.json") rescue nil
+      package_name = "General/#{package['name']}"
+      slug = 'stable'
+      json = get_json("#{@registry_url}/docs/#{package_name}/#{slug}/pkg.json") rescue nil
       json = {} if json.nil?
       {
         name: package['name'],
-        description: package['metadata']['description'],
+        description: json['description'],
         homepage: json['homepage'],
-        repository_url: repo_fallback(package['metadata']['repo'], json['homepage']),
-        keywords_array: package['metadata']['tags'],
-        versions: package['metadata']['versions'],
-        licenses: package['license'],
+        repository_url: repo_fallback(json['url'], json['homepage']),
+        keywords_array: json['tags'],
+        licenses: json['license'],
         downloads: fetch_downloads(package),
         downloads_period: 'total',
         metadata: {
-          uuid: package['uuid'],
-          slug: slug
+          uuid: json['uuid']
         }
       }
     end
@@ -106,36 +99,24 @@ def fetch_downloads(package)
     end
 
     def versions_metadata(pkg_metadata, existing_version_numbers = [])
-      begin
-        repo_json = get_json("https://repos.ecosyste.ms/api/v1/repositories/lookup?url=#{CGI.escape(pkg_metadata[:repository_url])}")
-        tags_json = get_json("https://repos.ecosyste.ms/api/v1/hosts/#{repo_json['host']['name']}/repositories/#{repo_json['full_name']}/tags")
-      rescue
-        tags_json = []
-      end
-      pkg_metadata[:versions].map do |v|
-        hash = {
-          number: v,
-          published_at: nil,
-          metadata: {}
+      version_numbers = get_json("#{@registry_url}/docs/General/#{pkg_metadata[:name]}/versions.json")
+      version_numbers.map do |v|
+        version_json = get_json("#{@registry_url}/docs/General/#{pkg_metadata[:name]}/#{v}/pkg.json")
+        next if version_json.nil?
+        {
+          number: version_json['version'],
+          published_at: version_json['release_date'],
+          licenses: version_json['license'],
+          metadata: {
+            slug: version_json['slug'],
+            uuid: version_json['uuid'],
+          }
         }
-
-        if tags_json.any?
-          tag = tags_json.find{|t| t['name'].to_s.downcase.delete_prefix('v') == v}
-          if tag
-            hash[:published_at] = tag['published_at']
-            hash[:metadata] = {
-              sha: tag['sha'],
-              download_url: tag['download_url']
-            }
-          end
-        end
-
-        hash
-      end
+      end.compact
     end
 
     def dependencies_metadata(name, version, package)
-      json = get_json("#{@registry_url}/docs/#{package[:name]}/#{package[:metadata][:slug]}/#{version}/pkg.json")
+      json = get_json("#{@registry_url}/docs/General/#{package[:name]}/#{version}/pkg.json")
       json['deps'].map do |dep|
         next if dep['direct'] == false
         next if dep['versions'].join(',') == '*' # skip std libraries

app/services/url_parser.rb

@@ -4,7 +4,7 @@ def self.parse(url)
   end
 
   def initialize(url)
-    @url = url.to_s
+    @url = url.to_s.dup
   end
 
   def parse
@@ -103,50 +103,50 @@ def extractable_early?
   end
 
   def remove_anchors
-    url.gsub!(/(#\S*)$/i, '')
+    self.url = url.dup.gsub(/(#\S*)$/i, '')
   end
 
   def remove_auth_user
-    self.url = url.split('@')[-1]
+    self.url = url.dup.split('@')[-1]
   end
 
   def remove_domain
     raise NotImplementedError
   end
 
   def remove_brackets
-    url.gsub!(/>|<|\(|\)|\[|\]/, '')
+    self.url = url.dup.gsub(/>|<|\(|\)|\[|\]/, '')
   end
 
   def remove_equals_sign
-    self.url = url.split('=')[-1]
+    self.url = url.dup.split('=')[-1]
   end
 
   def remove_extra_segments
-    self.url = url.split('/').reject(&:blank?)[0..1]
+    self.url = url.dup.split('/').reject(&:blank?)[0..1]
   end
 
   def remove_git_extension
-    url.gsub!(/(\.git|\/)$/i, '')
+    self.url = url.dup.gsub(/(\.git|\/)$/i, '')
   end
 
   def remove_git_scheme
-    url.gsub!(/git\/\//i, '')
+    self.url = url.dup.gsub(/git\/\//i, '')
   end
 
   def remove_querystring
-    url.gsub!(/(\?\S*)$/i, '')
+    self.url = url.dup.gsub(/(\?\S*)$/i, '')
   end
 
   def remove_scheme
-    url.gsub!(/(((git\+https|git|ssh|hg|svn|scm|http|https)+?:)+?)/i, '')
+    self.url = url.dup.gsub(/(((git\+https|git|ssh|hg|svn|scm|http|https)+?:)+?)/i, '')
   end
 
   def remove_subdomain
-    url.gsub!(/(www|ssh|raw|git|wiki)+?\./i, '')
+    self.url = url.dup.gsub(/(www|ssh|raw|git|wiki)+?\./i, '')
   end
 
   def remove_whitespace
-    url.gsub!(/\s/, '')
+    self.url = url.dup.gsub(/\s/, '')
   end
-end
+end

config/application.rb

@@ -23,7 +23,8 @@ module Packages
   class Application < Rails::Application
     # Initialize configuration defaults for originally generated Rails version.
     config.load_defaults 7.0
-    config.exceptions_app = routes
+    config.exceptions_app = self.routes
+    config.active_support.to_time_preserves_timezone = :zone
 
     # Configuration for the application, engines, and railties goes here.
     #

config/initializers/faraday.rb

@@ -1,2 +1,2 @@
-# require 'faraday/typhoeus'
-# Faraday.default_adapter = :typhoeus
+require 'faraday/typhoeus'
+Faraday.default_adapter = :typhoeus

test/fixtures/files/julia/versions.json

@@ -0,0 +1 @@
+["0.0.4"]

test/models/ecosystem/julia_test.rb

@@ -10,12 +10,12 @@ class JuliaTest < ActiveSupport::TestCase
 
   test 'registry_url' do
     registry_url = @ecosystem.registry_url(@package)
-    assert_equal registry_url, 'https://juliahub.com/ui/Packages/Inequality/xDAp7/'
+    assert_equal registry_url, "https://juliahub.com/ui/Packages/General/Inequality/"
   end
 
   test 'registry_url with version' do
     registry_url = @ecosystem.registry_url(@package, @version)
-    assert_equal registry_url, 'https://juliahub.com/ui/Packages/Inequality/xDAp7/0.0.4'
+    assert_equal registry_url, "https://juliahub.com/ui/Packages/General/Inequality/0.0.4"
   end
 
   test 'download_url' do
@@ -25,12 +25,12 @@ class JuliaTest < ActiveSupport::TestCase
 
   test 'documentation_url' do
     documentation_url = @ecosystem.documentation_url(@package)
-    assert_equal documentation_url, 'https://docs.juliahub.com/Inequality/xDAp7/'
+    assert_nil documentation_url
   end
 
   test 'documentation_url with version' do
     documentation_url = @ecosystem.documentation_url(@package, @version.number)
-    assert_equal documentation_url, 'https://docs.juliahub.com/Inequality/xDAp7/0.0.4'
+    assert_nil documentation_url
   end
 
   test 'install_command' do
@@ -40,12 +40,12 @@ class JuliaTest < ActiveSupport::TestCase
 
   test 'install_command with version' do
     install_command = @ecosystem.install_command(@package, @version.number)
-    assert_equal install_command, 'Pkg.add(Pkg.PackageSpec(;name="Inequality", version="0.0.4"))'
+    assert_equal install_command, "Pkg.add(\"Inequality@0.0.4\")"
   end
 
   test 'check_status_url' do
     check_status_url = @ecosystem.check_status_url(@package)
-    assert_equal check_status_url, "https://juliahub.com/docs/Inequality/xDAp7/pkg.json"
+    assert_equal check_status_url, "https://juliahub.com/docs/General/Inequality/stable/pkg.json"
   end
 
   test 'purl' do
@@ -79,7 +79,7 @@ class JuliaTest < ActiveSupport::TestCase
   test 'package_metadata' do
     stub_request(:get, "https://juliahub.com/app/packages/info")
       .to_return({ status: 200, body: file_fixture('julia/info') })
-    stub_request(:get, "https://juliahub.com/docs/Inequality/xDAp7/pkg.json")
+    stub_request(:get, "https://juliahub.com/docs/General/Inequality/stable/pkg.json")
       .to_return({ status: 200, body: file_fixture('julia/pkg.json') })
     stub_request(:get, "https://pkgs.genieframework.com/api/v1/badge/Inequality")
       .to_return({ status: 200, body: file_fixture('julia/Inequality') })
@@ -103,31 +103,37 @@ class JuliaTest < ActiveSupport::TestCase
       .to_return({ status: 200, body: file_fixture('julia/lookup?url=https:%2F%2Fgithub.com%2FJosepER%2FInequality.jl') })
     stub_request(:get, "https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/JosepER/Inequality.jl/tags")
       .to_return({ status: 200, body: file_fixture('julia/tags') })
-    stub_request(:get, "https://juliahub.com/docs/Inequality/xDAp7/pkg.json")
+    stub_request(:get, "https://juliahub.com/docs/General/Inequality/stable/pkg.json")
       .to_return({ status: 200, body: file_fixture('julia/pkg.json') })
+    stub_request(:get, "https://juliahub.com/docs/General/Inequality/versions.json")
+      .to_return({ status: 200, body: file_fixture('julia/versions.json') })
+    stub_request(:get, "https://juliahub.com/docs/General/Inequality/0.0.4/pkg.json")
+      .to_return({ status: 200, body: file_fixture('julia/0.0.4.pkg.json') })
     package_metadata = @ecosystem.package_metadata('Inequality')
+
     versions_metadata = @ecosystem.versions_metadata(package_metadata)
 
-    assert_equal versions_metadata, [{:number=>"0.0.4", :published_at=>"2022-04-05T14:38:12.000Z", :metadata=>{:sha=>"5b80978727dd84af7978b803d9e1e4c8ae7f75e8", :download_url=>"https://codeload.github.com/JosepER/Inequality.jl/tar.gz/v0.0.4"}}]
+    assert_equal versions_metadata, [{number: "0.0.4", published_at: "Apr 2022", licenses: "MIT", metadata: {slug: "xDAp7", uuid: "131cd6a8-f02b-4a66-8ce5-0a80ec47b73f"}}]
   end
 
   test 'dependencies_metadata' do
     stub_request(:get, "https://juliahub.com/app/packages/info")
       .to_return({ status: 200, body: file_fixture('julia/info') })
     stub_request(:get, "https://pkgs.genieframework.com/api/v1/badge/Inequality")
       .to_return({ status: 200, body: file_fixture('julia/Inequality') })
-    stub_request(:get, "https://juliahub.com/docs/Inequality/xDAp7/pkg.json")
+    stub_request(:get, "https://juliahub.com/docs/General/Inequality/stable/pkg.json")
       .to_return({ status: 200, body: file_fixture('julia/pkg.json') })
-    stub_request(:get, "https://juliahub.com/docs/Inequality/xDAp7/0.0.4/pkg.json")
+    stub_request(:get, "https://juliahub.com/docs/General/Inequality/0.0.4/pkg.json")
       .to_return({ status: 200, body: file_fixture('julia/0.0.4.pkg.json') })
     package_metadata = @ecosystem.package_metadata('Inequality')
     dependencies_metadata = @ecosystem.dependencies_metadata('Inequality', '0.0.4', package_metadata)
 
     assert_equal dependencies_metadata, [
-      {:package_name=>"DataFrames", :requirements=>"1.3.0-1", :kind=>"runtime", :ecosystem=>"julia"},
-      {:package_name=>"Documenter", :requirements=>"0.27", :kind=>"runtime", :ecosystem=>"julia"},
-      {:package_name=>"StatsBase", :requirements=>"0.33", :kind=>"runtime", :ecosystem=>"julia"},
-      {:package_name=>"julia", :requirements=>"1", :kind=>"runtime", :ecosystem=>"julia"}
+      {package_name: "DataFrames", requirements: "1.3.0-1", kind: "runtime", ecosystem: "julia"},
+      {package_name: "Documenter", requirements: "0.27", kind: "runtime", ecosystem: "julia"},
+      {package_name: "Statistics", requirements: "1", kind: "runtime", ecosystem: "julia"},
+      {package_name: "StatsBase", requirements: "0.33", kind: "runtime", ecosystem: "julia"},
+      {package_name: "julia", requirements: "1", kind: "runtime", ecosystem: "julia"}
     ]
   end
 end

test/tasks/packages_rake_test.rb

@@ -3,7 +3,7 @@
 
 class PackagesRakeTest < ActiveSupport::TestCase
   setup do
-    Packages::Application.load_tasks if Rake::Task.tasks.empty?
+    Packages::Application.load_tasks if Rake::Task.tasks.empty? 
   end
 
   test "should sync recent packages" do