diff --git a/.statbus b/.statbus new file mode 100644 index 000000000..e69de29bb diff --git a/cli/shard.lock b/cli/shard.lock index 8ff1dd590..be48e6e47 100644 --- a/cli/shard.lock +++ b/cli/shard.lock @@ -1,5 +1,9 @@ version: 2.0 shards: + bindata: + git: https://github.com/spider-gazelle/bindata.git + version: 2.1.0 + commander: git: https://github.com/mrrooijen/commander.git version: 0.4.0 @@ -8,6 +12,14 @@ shards: git: https://github.com/crystal-lang/crystal-db.git version: 0.12.0 + jwt: + git: https://github.com/crystal-community/jwt.git + version: 1.6.1 + + openssl_ext: + git: https://github.com/spider-gazelle/openssl_ext.git + version: 2.4.4 + pg: git: https://github.com/17dec/crystal-pg.git version: 0.27.0+git.commit.ed26146a226f31acf77c1949dea6d3e7c6b5674e diff --git a/cli/shard.yml b/cli/shard.yml index 647cd28dd..61cb76904 100644 --- a/cli/shard.yml +++ b/cli/shard.yml @@ -15,6 +15,8 @@ crystal: ">= 1.10.1" license: MIT dependencies: + jwt: + github: crystal-community/jwt pg: github: 17dec/crystal-pg branch: copyout diff --git a/cli/spec/dotenv_spec.cr b/cli/spec/dotenv_spec.cr index b9bec4035..9821e823a 100644 --- a/cli/spec/dotenv_spec.cr +++ b/cli/spec/dotenv_spec.cr @@ -5,7 +5,7 @@ private def with_tempfile(content : String) tempfile = File.tempfile(".env-test") begin File.write(tempfile.path, content) - yield tempfile.path + yield Path.new(tempfile.path) ensure tempfile.delete end @@ -43,8 +43,8 @@ describe Dotenv do with_tempfile(content) do |path| dotenv = Dotenv.from_file(path) - dotenv.env_file.lines[0].should be_a(Dotenv::CommentLine) - dotenv.env_file.lines[2].should be_a(Dotenv::CommentLine) + dotenv.dotenv_content.lines[0].should be_a(Dotenv::CommentLine) + dotenv.dotenv_content.lines[2].should be_a(Dotenv::CommentLine) end end @@ -57,7 +57,7 @@ describe Dotenv do with_tempfile(content) do |path| dotenv = Dotenv.from_file(path) - dotenv.env_file.lines[1].should be_a(Dotenv::BlankLine) + dotenv.dotenv_content.lines[1].should be_a(Dotenv::BlankLine) end end @@ -66,7 +66,7 @@ describe Dotenv do with_tempfile(content) do |path| dotenv = Dotenv.from_file(path) - line = dotenv.env_file.lines[0].as(Dotenv::KeyValueLine) + line = dotenv.dotenv_content.lines[0].as(Dotenv::KeyValueLine) line.key.should eq("KEY") line.value.should eq("value") line.inline_comment.should eq(" # inline comment") @@ -77,7 +77,7 @@ describe Dotenv do with_tempfile(content) do |path| dotenv = Dotenv.from_file(path) - line = dotenv.env_file.lines[0].as(Dotenv::KeyValueLine) + line = dotenv.dotenv_content.lines[0].as(Dotenv::KeyValueLine) line.key.should eq("KEY") line.value.should eq("value") line.inline_comment.should eq(" # inline comment") @@ -204,32 +204,32 @@ describe Dotenv do dotenv = Dotenv.from_file(path) # Verify specific aspects - dotenv.env_file.lines[0].should be_a(Dotenv::CommentLine) - dotenv.env_file.lines[1].should be_a(Dotenv::BlankLine) + dotenv.dotenv_content.lines[0].should be_a(Dotenv::CommentLine) + dotenv.dotenv_content.lines[1].should be_a(Dotenv::BlankLine) - empty = dotenv.env_file.lines[4].as(Dotenv::KeyValueLine) + empty = dotenv.dotenv_content.lines[4].as(Dotenv::KeyValueLine) empty.key.should eq("EMPTY") empty.value.should eq("") - spaces = dotenv.env_file.lines[5].as(Dotenv::KeyValueLine) + spaces = dotenv.dotenv_content.lines[5].as(Dotenv::KeyValueLine) spaces.key.should eq("SPACES") spaces.value.should eq(" spaced value") spaces.inline_comment.should eq(" # With trailing comment") - quotes = dotenv.env_file.lines[6].as(Dotenv::KeyValueLine) + quotes = dotenv.dotenv_content.lines[6].as(Dotenv::KeyValueLine) quotes.key.should eq("QUOTES") quotes.value.should eq("quoted value") quotes.quote.should eq('"') - escaped = dotenv.env_file.lines[7].as(Dotenv::KeyValueLine) + escaped = dotenv.dotenv_content.lines[7].as(Dotenv::KeyValueLine) escaped.key.should eq("ESCAPED") escaped.value.should eq("escaped\\\"quote") - newlines = dotenv.env_file.lines[8].as(Dotenv::KeyValueLine) + newlines = dotenv.dotenv_content.lines[8].as(Dotenv::KeyValueLine) newlines.key.should eq("NEWLINES") newlines.value.should eq("multi\\nline") - inline = dotenv.env_file.lines[9].as(Dotenv::KeyValueLine) + inline = dotenv.dotenv_content.lines[9].as(Dotenv::KeyValueLine) inline.key.should eq("KEY") inline.value.should eq("value") inline.inline_comment.should eq(" # inline comment") @@ -242,36 +242,54 @@ describe Dotenv do reloaded = Dotenv.from_file(path) # Verify file contents are identical - reloaded.env_file.to_s.should eq(content) + reloaded.dotenv_content.to_s.should eq(content) end end end describe ".using" do - it "automatically saves changes" do + it "automatically saves changes when using Path" do with_tempfile("KEY=value") do |path| initial_content = File.read(path) - + Dotenv.using(path) do |dotenv| dotenv.set("KEY", "new_value") end - + File.read(path).should_not eq(initial_content) File.read(path).should contain("KEY=new_value") end end - it "doesn't save if no changes" do + it "doesn't save if no changes when using Path" do with_tempfile("KEY=value") do |path| initial_content = File.read(path) - + Dotenv.using(path) do |dotenv| # Just read, no changes dotenv.get("KEY") end - + File.read(path).should eq(initial_content) end end + + it "works with string content" do + content = "KEY=value" + result = Dotenv.using(content) do |dotenv| + dotenv.get("KEY").should eq("value") + dotenv.set("NEW_KEY", "new_value") + "return value" + end + result.should eq("return value") + end + + it "doesn't try to save when using string content" do + content = "KEY=value" + Dotenv.using(content) do |dotenv| + dotenv.set("KEY", "new_value") + # No file should be created/modified + end + end end end diff --git a/cli/src/dotenv.cr b/cli/src/dotenv.cr index 3aa3f96fd..da01af571 100644 --- a/cli/src/dotenv.cr +++ b/cli/src/dotenv.cr @@ -38,8 +38,6 @@ require "file_utils" # - Leading whitespace # - Original line ordering class Dotenv - @@key : String | Nil = nil - class Error < Exception; end # Represents different types of lines in a .env file @@ -92,7 +90,7 @@ class Dotenv end # Represents the entire .env file - class EnvFile + class DotenvContent property lines : Array(EnvLine) property mapping : Hash(String, KeyValueLine) @@ -105,7 +103,9 @@ class Dotenv @mapping[key]?.try(&.value) end - def set(key : String, value : String) + # Sets or updates a dotenv variable value + # Returns the value that was set + def set(key : String, value : String) : String if line = @mapping[key]? # Update existing line line.value = value @@ -116,6 +116,7 @@ class Dotenv @lines << line @mapping[key] = line end + value end def parse(keys : Array(String) = [] of String) : Hash(String, String) @@ -133,28 +134,50 @@ class Dotenv end end - property env_file : EnvFile - property dotenv_file : String + property dotenv_content : DotenvContent + property dotenv_path : Path? property verbose : Bool - def initialize(@dotenv_file = ".env", @verbose = false) - @env_file = EnvFile.new - load_file + def initialize(dotenv_path : String | Path | Nil = ".env", content : String? = nil, @verbose = false) + @dotenv_path = dotenv_path.try { |path| path.is_a?(Path) ? path : Path.new(path) } + @dotenv_content = DotenvContent.new + + if content + parse_content(content) + elsif dotenv_path + load_file + end + end + + def self.from_file(file : String | Path, verbose = false) : Dotenv + new(dotenv_path: file, verbose: verbose) end - def self.from_file(file : String, verbose = false) : Dotenv - new(file, verbose) + def self.from_string(content : String, verbose = false) : Dotenv + new(dotenv_path: nil, content: content, verbose: verbose) end - # Opens a .env file, yields it to the block, and saves it if modified - def self.using(file : String, verbose = false) - dotenv = from_file(file, verbose) - initial_content = dotenv.env_file.to_s - yield dotenv - # Only save if content changed - if dotenv.env_file.to_s != initial_content + # Opens a .env file or parses content, yields it to the block, and saves if modified + # and we have a Path. + def self.using(source : Path | String, verbose = false, &block : Dotenv -> T) forall T + dotenv = case source + when Path + from_file(source, verbose) + when String + from_string(source, verbose) + else + raise "Unsupported source type" + end + + initial_content = dotenv.dotenv_content.to_s + result = block.call(dotenv) + + # Only save if content changed and we're working with a file + if source.is_a?(Path) && dotenv.dotenv_content.to_s != initial_content dotenv.save_file end + + return result end # Parses a line into an appropriate EnvLine object @@ -186,50 +209,59 @@ class Dotenv end end - # Appends a line to the .env file and updates the in-memory representation + # Appends a line to the dotenv contents and updates the in-memory representation def puts(line : String) env_line = parse_line(line) if env_line.is_a?(KeyValueLine) STDERR.puts "Adding line: #{env_line.key}=#{env_line.value}" if @verbose - File.open(@dotenv_file, "a") do |file| + File.open(@dotenv_path, "a") do |file| file.puts(line) end - @env_file.lines << env_line - @env_file.mapping[env_line.key] = env_line + @dotenv_content.lines << env_line + @dotenv_content.mapping[env_line.key] = env_line else - File.open(@dotenv_file, "a") do |file| + File.open(@dotenv_path, "a") do |file| file.puts(line) end - @env_file.lines << env_line + @dotenv_content.lines << env_line end end # Loads and parses the .env file into memory - def load_file - return unless File.exists?(@dotenv_file) - - STDERR.puts "Loading #{@dotenv_file}" if @verbose - @env_file = EnvFile.new + # Parses content from a string + def parse_content(content : String) + STDERR.puts "Parsing content" if @verbose + @dotenv_content = DotenvContent.new - File.each_line(@dotenv_file) do |line| + content.each_line do |line| env_line = parse_line(line) - @env_file.lines << env_line + @dotenv_content.lines << env_line if env_line.is_a?(KeyValueLine) - STDERR.puts "Loaded: #{env_line.key}=#{env_line.value}" if @verbose - @env_file.mapping[env_line.key] = env_line + STDERR.puts "Parsed: #{env_line.key}=#{env_line.value}" if @verbose + @dotenv_content.mapping[env_line.key] = env_line end end - STDERR.puts "Finished loading #{@dotenv_file}" if @verbose + end + + # Loads and parses from a file + def load_file + return unless @dotenv_path && File.exists?(@dotenv_path.not_nil!) + + STDERR.puts "Loading #{@dotenv_path}" if @verbose + parse_content(File.read(@dotenv_path.not_nil!)) + STDERR.puts "Finished loading #{@dotenv_path}" if @verbose end def save_file - STDERR.puts "Saving to #{@dotenv_file}" if @verbose - File.write(@dotenv_file, @env_file.to_s) - STDERR.puts "Saved #{@env_file.mapping.size} entries" if @verbose + raise "Cannot save - no path specified" unless @dotenv_path + + STDERR.puts "Saving to #{@dotenv_path}" if @verbose + File.write(@dotenv_path.not_nil!, @dotenv_content.to_s) + STDERR.puts "Saved #{@dotenv_content.mapping.size} entries" if @verbose end def get(key : String) : String? - value = @env_file.get(key) + value = @dotenv_content.get(key) STDERR.puts "Getting #{key}=#{value}" if @verbose return value end @@ -240,35 +272,39 @@ class Dotenv STDERR.puts "Setting #{key}" if @verbose if value.starts_with?("+") default_value = value[1..-1] - if @env_file.mapping.has_key?(key) + if @dotenv_content.mapping.has_key?(key) STDERR.puts "Key #{key} exists, keeping current value" if @verbose else STDERR.puts "Key #{key} not found, setting default: #{default_value}" if @verbose - @env_file.set(key, default_value) + @dotenv_content.set(key, default_value) end else STDERR.puts "Setting #{key}=#{value}" if @verbose - @env_file.set(key, value) + @dotenv_content.set(key, value) end - save_file end def export - @env_file.mapping.each { |k, v| ENV[k] = v.value } + @dotenv_content.mapping.each { |k, v| ENV[k] = v.value } end def parse(keys : Array(String) = [] of String) - @env_file.parse(keys) + @dotenv_content.parse(keys) end - def generate(key : String, &block : -> String) - return if @env_file.mapping[key]? - value = block.call - @env_file.set(key, value) - save_file + # Generates a value for a key if it doesn't exist + # Returns the generated value or the existing value + def generate(key : String, &block : -> String) : String + if existing = @dotenv_content.mapping[key]? + existing.value + else + value = block.call + @dotenv_content.set(key, value) + value + end end - def self.run(dotenv_file : String = ".env") + def self.run(dotenv_filename : String = ".env") cli = Commander::Command.new do |cmd| cmd.use = "dotenv" cmd.long = "Manage .env files containing environment variables" @@ -323,6 +359,7 @@ class Dotenv exit(1) else dotenv.set(key, value) + dotenv.save_file end end end @@ -363,6 +400,7 @@ class Dotenv exit(1) else dotenv.generate(key) { `#{command}`.strip } + dotenv.save_file end end end diff --git a/cli/src/statbus.cr b/cli/src/statbus.cr index d69779254..ee789a35b 100644 --- a/cli/src/statbus.cr +++ b/cli/src/statbus.cr @@ -1,6 +1,7 @@ require "http/client" require "json" require "digest/sha256" +require "./dotenv" require "time" require "option_parser" require "dir" @@ -10,6 +11,7 @@ require "pg" require "file" require "csv" require "yaml" +require "jwt" # The `Statbus` module is designed to manage and import data for a statistical business registry. # It supports various operations like installation, management (start, stop, status), and data import. @@ -30,6 +32,7 @@ class StatBus Stop Status CreateUsers + GenerateConfig end enum ImportStrategy Copy @@ -146,7 +149,7 @@ class StatBus @refresh_materialized_views = true @import_file_name : String | Nil = nil @config_field_mapping = Array(ConfigFieldMapping).new - @config_file_path : Path | Nil = nil + @config_field_mapping_file_path : Path | Nil = nil @sql_field_mapping = Array(SqlFieldMapping).new @working_directory = Dir.current @project_directory : Path @@ -154,7 +157,7 @@ class StatBus private def initialize_project_directory : Path # First try from current directory current = Path.new(Dir.current) - found = find_env_in_parents(current) + found = find_statbus_in_parents(current) if found.nil? # Fall back to executable path @@ -163,17 +166,18 @@ class StatBus current # Last resort: use current dir else exec_dir = Path.new(Path.new(executable_path).dirname) - find_env_in_parents(exec_dir) || current + find_statbus_in_parents(exec_dir) || current end else found end end - private def find_env_in_parents(start_path : Path) : Path? + private def find_statbus_in_parents(start_path : Path) : Path? current = start_path while current.to_s != "/" - if File.exists?(current.join(".env")) + # The .statbus is an empty marker file placed in the statbus directory. + if File.exists?(current.join(".statbus")) return current end current = current.parent @@ -212,13 +216,9 @@ class StatBus Dir.cd(@project_directory) do if File.exists? ".env" puts "The config is already generated" - elsif File.exists? ".env.example" - puts "Generating a new config file" - # Read .env.example - # Generate random secrets and JWT's - # Write .env else puts "Could not find template for .env" + manage_generate_config end end puts "installed" @@ -376,10 +376,10 @@ class StatBus # This mapping can be printed for the user. puts "Example mapping:" Dir.cd(@working_directory) do - if !@config_file_path.nil? - if !File.exists?(@config_file_path.not_nil!) + if !@config_field_mapping_file_path.nil? + if !File.exists?(@config_field_mapping_file_path.not_nil!) File.write( - @config_file_path.not_nil!, + @config_field_mapping_file_path.not_nil!, @config_field_mapping.to_pretty_json ) end @@ -392,11 +392,11 @@ class StatBus puts "@config_field_mapping = #{@config_field_mapping}" if @verbose Dir.cd(@working_directory) do - if !@config_file_path.nil? - if !File.exists?(@config_file_path.not_nil!) - puts "Writing file #{@config_file_path}" + if !@config_field_mapping_file_path.nil? + if !File.exists?(@config_field_mapping_file_path.not_nil!) + puts "Writing file #{@config_field_mapping_file_path}" File.write( - @config_file_path.not_nil!, + @config_field_mapping_file_path.not_nil!, @config_field_mapping.to_pretty_json ) puts @config_field_mapping.to_pretty_json @@ -626,6 +626,9 @@ class StatBus parser.on("create-users", "Create users from .users.yml file") do @manage_mode = ManageMode::CreateUsers end + parser.on("generate-config", "Generate configuration files") do + @manage_mode = ManageMode::GenerateConfig + end end parser.on("import", "Import into installed StatBus") do @mode = Mode::Import @@ -663,13 +666,13 @@ class StatBus end parser.on("-c FILENAME", "--config=FILENAME", "A config file with field mappings. Will be written to with an example if the file does not exist.") do |file_name| Dir.cd(@working_directory) do - @config_file_path = Path.new(file_name) - if File.exists?(@config_file_path.not_nil!) + @config_field_mapping_file_path = Path.new(file_name) + if File.exists?(@config_field_mapping_file_path.not_nil!) puts "Loading mapping from #{file_name}" - config_data = File.read(@config_file_path.not_nil!) + config_data = File.read(@config_field_mapping_file_path.not_nil!) @config_field_mapping = Array(ConfigFieldMapping).from_json(config_data) else - STDERR.puts "Could not find #{@config_file_path}" + STDERR.puts "Could not find #{@config_field_mapping_file_path}" end end end @@ -884,6 +887,8 @@ class StatBus manage_status when ManageMode::CreateUsers create_users + when ManageMode::GenerateConfig + manage_generate_config else puts "Unknown manage mode #{@manage_mode}" # puts parser @@ -1145,6 +1150,351 @@ class StatBus end end + # Ref. https://forum.crystal-lang.org/t/is-this-a-good-way-to-generate-a-random-string/6986/2 + private def random_string(len) : String + chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + String.new(len) do |bytes| + bytes.to_slice(len).fill { chars.to_slice.sample } + {len, len} + end + end + + # Type-safe credentials structure + record CredentialsEnv, + postgres_password : String, + jwt_secret : String, + dashboard_username : String, + dashboard_password : String, + anon_key : String, + service_role_key : String + + # Type-safe configuration structure + record ConfigEnv, + deployment_slot_name : String, + deployment_slot_code : String, + deployment_slot_port_offset : String, + statbus_url : String, + browser_supabase_url : String, + server_supabase_url : String, + seq_server_url : String, + seq_api_key : String, + slack_token : String + + # Configuration values that are derived from other settings + record DerivedEnv, + app_bind_address : String, + supabase_bind_address : String, + db_public_localhost_port : String, + version : String, + site_url : String, + api_external_url : String, + supabase_public_url : String, + enable_email_signup : Bool, + enable_email_autoconfirm : Bool, + disable_signup : Bool, + studio_default_project : String + + private def manage_generate_config + Dir.cd(@project_directory) do + credentials_file = Path.new(".env.credentials") + if File.exists?(credentials_file) + puts "Using existing credentials from #{credentials_file}" if @verbose + else + puts "Generating new credentials in #{credentials_file}" if @verbose + end + + # Generate or read existing credentials + + credentials = Dotenv.using(credentials_file) do |credentials_env| + jwt_secret = credentials_env.generate("JWT_SECRET") { random_string(32) } + + # Generate JWT tokens + # While the JWT tokens are calculated, the way Supabase is configured, it seems it does a TEXTUAL + # equality check of the ANON JWT, and not an actual secret based calculation. + # so if the JWT token is generated again, with a different timestamp, even if it is signed + # with the same secret, it fails. + # Therefore we store the derived JWT tokens as a credential, because it can not change without + # invalidating the deployed or copied tokens. 🤦‍♂️ + + # Issued At Time: Current timestamp in seconds since the Unix epoch + iat = Time.utc.to_unix + # Expiration Time: Calculate exp as iat plus the seconds in 5 years + exp = iat + (5 * 365 * 24 * 60 * 60) # 5 years + + anon_payload = { + role: "anon", + iss: "supabase", + iat: iat, + exp: exp, + } + + service_role_payload = { + role: "service_role", + iss: "supabase", + iat: iat, + exp: exp, + } + + anon_key = JWT.encode(anon_payload, jwt_secret, JWT::Algorithm::HS256) + service_role_key = JWT.encode(service_role_payload, jwt_secret, JWT::Algorithm::HS256) + + CredentialsEnv.new( + postgres_password: credentials_env.generate("POSTGRES_PASSWORD") { random_string(20) }, + jwt_secret: jwt_secret, + dashboard_username: credentials_env.generate("DASHBOARD_USERNAME") { "admin" }, + dashboard_password: credentials_env.generate("DASHBOARD_PASSWORD") { random_string(20) }, + anon_key: credentials_env.generate("ANON_KEY") { anon_key }, + service_role_key: credentials_env.generate("SERVICE_ROLE_KEY") { service_role_key }, + ) + end + + # Load or generate config + config_file = Path.new(".env.config") + if File.exists?(config_file) + puts "Using existing config from #{config_file}" if @verbose + else + puts "Generating new config in #{config_file}" if @verbose + end + + config = Dotenv.using(config_file) do |config_env| + ConfigEnv.new( + deployment_slot_name: config_env.generate("DEPLOYMENT_SLOT_NAME") { "Development" }, + deployment_slot_code: config_env.generate("DEPLOYMENT_SLOT_CODE") { "dev" }, + deployment_slot_port_offset: config_env.generate("DEPLOYMENT_SLOT_PORT_OFFSET") { "1" }, + statbus_url: config_env.generate("STATBUS_URL") { "http://localhost:3010" }, + browser_supabase_url: config_env.generate("BROWSER_SUPABASE_URL") { "http://localhost:3011" }, + server_supabase_url: config_env.generate("SERVER_SUPABASE_URL") { "http://kong:8000" }, + seq_server_url: config_env.generate("SEQ_SERVER_URL") { "https://log.statbus.org" }, + seq_api_key: config_env.generate("SEQ_API_KEY") { "secret_seq_api_key" }, + slack_token: config_env.generate("SLACK_TOKEN") { "secret_slack_api_token" } + ) + end + + # Calculate derived values + base_port = 3000 + slot_multiplier = 10 + port_offset = base_port + (config.deployment_slot_port_offset.to_i * slot_multiplier) + + derived = DerivedEnv.new( + # The host address connected to the STATBUS app + app_bind_address: "127.0.0.1:#{port_offset}", + # The host address connected to Supabase + supabase_bind_address: "127.0.0.1:#{port_offset + 1}", + # The publicly exposed address of PostgreSQL inside Supabase + db_public_localhost_port: (port_offset + 2).to_s, + # Git version of the deployed commit + version: `git describe --always`.strip, + # URL where the site is hosted + site_url: config.statbus_url, + # External URL for the API + api_external_url: config.browser_supabase_url, + # Public URL for Supabase access + supabase_public_url: config.browser_supabase_url, + # Maps to GOTRUE_EXTERNAL_EMAIL_ENABLED to allow authentication with Email at all. + # So SIGNUP really means SIGNIN + enable_email_signup: true, + # Allow creating users and setting the email as verified, + # rather than sending an actual email where the user must + # click the link. + enable_email_autoconfirm: true, + # Disables signup with EMAIL, when ENABLE_EMAIL_SIGNUP=true + disable_signup: true, + # Sets the project name in the Supabase API portal + studio_default_project: config.deployment_slot_name + ) + + # Generate or update .env file + new_content = generate_env_content(credentials, config, derived) + if File.exists?(".env") + puts "Checking existing .env for changes" if @verbose + current_content = File.read(".env") + + if new_content != current_content + backup_suffix = Time.utc.to_s("%Y-%m-%d") + counter = 1 + while File.exists?(".env.backup.#{backup_suffix}") + backup_suffix = "#{Time.utc.to_s("%Y-%m-%d")}_#{counter}" + counter += 1 + end + + puts "Updating .env with changes - old version backed up as .env.backup.#{backup_suffix}" if @verbose + File.write(".env.backup.#{backup_suffix}", current_content) + File.write(".env", new_content) + return # Skip the File.write below since we already wrote the file + else + puts "No changes detected in .env, skipping backup" if @verbose + end + else + puts "Creating new .env file" if @verbose + File.write(".env", new_content) + end + + begin + # Generate Caddy configuration + deployment_slot_code = config.deployment_slot_code + deployment_user = "statbus_#{deployment_slot_code}" + domain = "#{deployment_slot_code}.statbus.org" + app_port = config.statbus_url.match(/:\d+/).not_nil![0].gsub(":", "") + api_port = config.browser_supabase_url.match(/:\d+/).not_nil![0].gsub(":", "") + + # Generate new Caddy content + new_caddy_content = generate_caddy_content( + deployment_user: deployment_user, + domain: domain, + app_port: app_port, + api_port: api_port + ) + + # Check if file exists and content differs + if File.exists?("deployment.caddyfile") + current_content = File.read("deployment.caddyfile") + if new_caddy_content != current_content + puts "Updating deployment.caddyfile with changes for #{domain}" if @verbose + File.write("deployment.caddyfile", new_caddy_content) + else + puts "No changes needed in deployment.caddyfile for #{domain}" if @verbose + end + else + puts "Creating new deployment.caddyfile for #{domain}" if @verbose + File.write("deployment.caddyfile", new_caddy_content) + end + end + end + end + + private def generate_env_content(credentials : CredentialsEnv, config : ConfigEnv, derived : DerivedEnv) : String + content = <<-EOS + ################################################################ + # Statbus Environment Variables + # Generated by `statbus manage generate-config` + # Used by docker compose, both for statbus containers + # and for the included supabase containers. + # The files: + # `.env.credentials` generated if missing, with stable credentials. + # `.env.config` generated if missing, configuration for installation. + # `.env` generated with input from `.env.credentials` and `.env.config` + # The `.env` file contains settings used both by + # the statbus app (Backend/frontend) and by the Supabase Docker + # containers. + # + # The top level `docker-compose.yml` file includes all configuration + # required for all statbus docker containers, but must be managed + # by `./devops/manage-statbus.sh` that also sets the VERSION + # required for precise logging by the statbus app. + ################################################################ + + ################################################################ + # Statbus Container Configuration + ################################################################ + + # The name displayed on the web + DEPLOYMENT_SLOT_NAME=#{config.deployment_slot_name} + # Urls configured in Caddy and DNS. + STATBUS_URL=#{config.statbus_url} + BROWSER_SUPABASE_URL=#{config.browser_supabase_url} + SERVER_SUPABASE_URL=#{config.server_supabase_url} + # Logging server + SEQ_SERVER_URL=#{config.seq_server_url} + SEQ_API_KEY=#{config.seq_api_key} + # Deployment Messages + SLACK_TOKEN=#{config.slack_token} + # The prefix used for all container names in docker + COMPOSE_INSTANCE_NAME=statbus-#{config.deployment_slot_code} + # The host address connected to the STATBUS app + APP_BIND_ADDRESS=#{derived.app_bind_address} + # The host address connected to Supabase + SUPABASE_BIND_ADDRESS=#{derived.supabase_bind_address} + # The publicly exposed address of PostgreSQL inside Supabase + DB_PUBLIC_LOCALHOST_PORT=#{derived.db_public_localhost_port} + # Updated by manage-statbus.sh start required + VERSION=#{derived.version} + EOS + content += "\n\n" + + supabase_env_filename = "supabase_docker/.env.example" + content += <<-EOS + ################################################################ + # Supabase Container Configuration + # Adapted from #{supabase_env_filename} + ################################################################ + EOS + content += "\n\n" + + # Add Supabase Docker content with overrides + supabase_env_path = Path.new(@project_directory, supabase_env_filename) + supabase_env_content = File.read(supabase_env_path) + content += Dotenv.using(supabase_env_content) do |env| + # Override credentials + env.set("POSTGRES_PASSWORD", credentials.postgres_password) + env.set("JWT_SECRET", credentials.jwt_secret) + env.set("ANON_KEY", credentials.anon_key) + env.set("SERVICE_ROLE_KEY", credentials.service_role_key) + env.set("DASHBOARD_USERNAME", credentials.dashboard_username) + env.set("DASHBOARD_PASSWORD", credentials.dashboard_password) + + # Set derived values + env.set("SITE_URL", derived.site_url) + env.set("API_EXTERNAL_URL", derived.api_external_url) + env.set("SUPABASE_PUBLIC_URL", derived.supabase_public_url) + env.set("ENABLE_EMAIL_SIGNUP", derived.enable_email_signup.to_s) + env.set("ENABLE_EMAIL_AUTOCONFIRM", derived.enable_email_autoconfirm.to_s) + env.set("DISABLE_SIGNUP", derived.disable_signup.to_s) + env.set("STUDIO_DEFAULT_PROJECT", derived.studio_default_project) + + # Return modified content without saving changes to example file + env.dotenv_content.to_s + end + content += "\n\n" + + content += <<-EOS + ################################################################ + # Statbus App Environment Variables + # Next.js only exposes environment variables with the 'NEXT_PUBLIC_' prefix + # to the browser cdoe. + # Add all the variables here that are exposed publicly, + # i.e. available in the web page source code for all to see. + # + NEXT_PUBLIC_SUPABASE_ANON_KEY=#{credentials.anon_key} + NEXT_PUBLIC_BROWSER_SUPABASE_URL=#{config.browser_supabase_url} + NEXT_PUBLIC_DEPLOYMENT_SLOT_NAME=#{config.deployment_slot_name} + NEXT_PUBLIC_DEPLOYMENT_SLOT_CODE=#{config.deployment_slot_code} + # + ################################################################ + EOS + + return content + end + + private def generate_caddy_content(deployment_user : String, domain : String, app_port : String, api_port : String) : String + <<-EOS + # Generated by statbus migrate generate-config + # Do not edit directly - changes will be lost + #{domain} { + redir https://www.#{domain} + } + + www.#{domain} { + @maintenance { + file { + try_files /home/#{deployment_user}/maintenance + } + } + handle @maintenance { + root * /home/#{deployment_user}/statbus/app/public + rewrite * /maintenance.html + file_server { + status 503 + } + } + reverse_proxy 127.0.0.1:#{app_port} + } + + api.#{domain} { + reverse_proxy 127.0.0.1:#{api_port} + } + EOS + end + private def cleanup_migration_schema(db) db.transaction do |tx| tx.connection.exec("DROP TABLE IF EXISTS db.migration") diff --git a/devops/manage-statbus.sh b/devops/manage-statbus.sh index b8e95dd51..975c84fde 100755 --- a/devops/manage-statbus.sh +++ b/devops/manage-statbus.sh @@ -309,263 +309,7 @@ case "$action" in git commit -m 'Upgraded Supabase Docker' ;; 'generate-config' ) - if ! test -f .users.yml; then - echo "Copy .users.example to .users.yml and add your admin users" - exit 1 - fi - - CREDENTIALS_FILE=".env.credentials" - echo Using credentials from $CREDENTIALS_FILE - POSTGRES_PASSWORD=$(./devops/dotenv --file $CREDENTIALS_FILE generate POSTGRES_PASSWORD pwgen 20) - JWT_SECRET=$(./devops/dotenv --file $CREDENTIALS_FILE generate JWT_SECRET pwgen 32) - DASHBOARD_USERNAME=$(./devops/dotenv --file $CREDENTIALS_FILE generate DASHBOARD_USERNAME echo admin) - DASHBOARD_PASSWORD=$(./devops/dotenv --file $CREDENTIALS_FILE generate DASHBOARD_PASSWORD pwgen 20) - - # While the JWT tokens are calculates, but the way Supabase is configured, it seems it does a TEXTUAL - # equality check of the ANON JWT, and not an actual secret based calculation. - # so if the JWT token is generated again, with a different timestamp, even if it is signed - # with the same secret, it fails. - # Therefore we store the derived JWT tokens as a credential, because it can not change without - # invalidating the deployed or copied tokens. 🤦‍♂️ - - # Issued At Time: Current timestamp in seconds since the Unix epoch - iat=$(date +%s) - # Number of seconds in 5 years (5 years * 365 days/year * 24 hours/day * 60 minutes/hour * 60 seconds/minute) - seconds_in_5_years=$((5 * 365 * 24 * 60 * 60)) - # Expiration Time: Calculate exp as iat plus the seconds in 5 years - exp=$((iat + seconds_in_5_years)) - jwt_anon_payload=$(cat </dev/null 2>&1; then - echo "Error: 'jwt' command not found. Please install jwt-cli with:" - echo " brew install mike-engel/jwt-cli/jwt-cli" - exit 1 - fi - - export ANON_KEY=$(jwt encode --secret "$JWT_SECRET" "$jwt_anon_payload") - ANON_KEY=$(./devops/dotenv --file $CREDENTIALS_FILE generate ANON_KEY echo $ANON_KEY) - - export SERVICE_ROLE_KEY=$(jwt encode --secret "$JWT_SECRET" "$jwt_service_role_payload") - SERVICE_ROLE_KEY=$(./devops/dotenv --file $CREDENTIALS_FILE generate SERVICE_ROLE_KEY echo $SERVICE_ROLE_KEY) - - CONFIG_FILE=".env.config" - echo Using config from $CONFIG_FILE - # The name displayed on the web - DEPLOYMENT_SLOT_NAME=$(./devops/dotenv --file $CONFIG_FILE generate DEPLOYMENT_SLOT_NAME echo "Development") - # Unique code used on the server for distinct docker namespaces - DEPLOYMENT_SLOT_CODE=$(./devops/dotenv --file $CONFIG_FILE generate DEPLOYMENT_SLOT_CODE echo "dev") - # Offset to calculate ports exposed by docker compose - DEPLOYMENT_SLOT_PORT_OFFSET=$(./devops/dotenv --file $CONFIG_FILE generate DEPLOYMENT_SLOT_PORT_OFFSET echo "1") - # Urls configured in Caddy and DNS. - STATBUS_URL=$(./devops/dotenv --file $CONFIG_FILE generate STATBUS_URL echo "http://localhost:3010") - BROWSER_SUPABASE_URL=$(./devops/dotenv --file $CONFIG_FILE generate BROWSER_SUPABASE_URL echo "http://localhost:3011") - SERVER_SUPABASE_URL=$(./devops/dotenv --file $CONFIG_FILE generate SERVER_SUPABASE_URL echo "http://kong:8000") - # Logging server - SEQ_SERVER_URL=$(./devops/dotenv --file $CONFIG_FILE generate SEQ_SERVER_URL echo "https://log.statbus.org") - SEQ_API_KEY=$(./devops/dotenv --file $CONFIG_FILE generate SEQ_API_KEY echo "secret_seq_api_key") - SLACK_TOKEN=$(./devops/dotenv --file $CONFIG_FILE generate SLACK_TOKEN echo "secret_slack_api_token") - - # Prepare a new environment file - # Check if the original file exists - if test -f .env; then - # Use the current date as the base for the backup suffix - backup_base=$(date -u +%Y-%m-%d) - backup_suffix="backup.$backup_base" - counter=1 - - # Loop to find a unique backup file name - while test -f ".env.$backup_suffix"; do - # If a file with the current suffix exists, increment the counter and append it to the suffix - backup_suffix="backup.${backup_base}_$counter" - ((counter++)) - done - - # Inform the user about the replacement and backup process - echo "Replacing .env - the old version is backed up as .env.$backup_suffix" - - # Move the original file to its backup location with the unique suffix - mv .env ".env.$backup_suffix" - fi - - cat > .env <<'EOS' -################################################################ -# Statbus Environment Variables -# Generated by `./devops/manage-statbus.sh generate-config` -# Used by docker compose, both for statbus containers -# and for the included supabase containers. -# The files: -# `.env.credentials` generated if missing, with stable credentials. -# `.env.config` generated if missing, configuration for installation. -# `.env` generated with input from `.env.credentials` and `.env.config` -# The `.env` file contains settings used both by -# the statbus app (Backend/frontend) and by the Supabase Docker -# containers. -# The top level `docker-compose.yml` file includes all configuration -# required for all statbus docker containers, but must be managed -# by `./devops/manage-statbus.sh` that also sets the VERSION -# required for precise logging by the statbus app. -################################################################ -EOS - - cat >> .env <<'EOS' - -################################################################ -# Statbus Container Configuration -################################################################ - -# The name displayed on the web -DEPLOYMENT_SLOT_NAME=Example -# Urls configured in Caddy and DNS. -STATBUS_URL=https://www.ex.statbus.org -BROWSER_SUPABASE_URL=https://api.ex.statbus.org -SERVER_SUPABASE_URL=http://kong:8000 -# Logging server -SEQ_SERVER_URL=https://log.statbus.org -SEQ_API_KEY=secret_seq_api_key -# Deployment Messages -SLACK_TOKEN=secret_slack_api_token -# The prefix used for all container names in docker -COMPOSE_INSTANCE_NAME=statbus -# The host address connected to the STATBUS app -APP_BIND_ADDRESS=127.0.0.1:3010 -# The host address connected to Supabase -SUPABASE_BIND_ADDRESS=127.0.0.1:3011 -# The publicly exposed address of PostgreSQL inside Supabase -DB_PUBLIC_LOCALHOST_PORT=3432 -# Updated by manage-statbus.sh start all -VERSION=commit_sha_or_version_of_deployed_commit -EOS - - cat >> .env <<'EOS' - -################################################################ -## Supabase Container Configuation -# Adapted from supabase_docker/.env.example -################################################################ - - -EOS - cat supabase_docker/.env.example >> .env - - echo "Setting Statbus Container Configuration" - ./devops/dotenv --file .env set DEPLOYMENT_SLOT_NAME="$DEPLOYMENT_SLOT_NAME" - ./devops/dotenv --file .env set COMPOSE_INSTANCE_NAME="statbus-$DEPLOYMENT_SLOT_CODE" - ./devops/dotenv --file .env set STATBUS_URL=$STATBUS_URL - ./devops/dotenv --file .env set BROWSER_SUPABASE_URL=$BROWSER_SUPABASE_URL - ./devops/dotenv --file .env set SERVER_SUPABASE_URL=$SERVER_SUPABASE_URL - ./devops/dotenv --file .env set SEQ_SERVER_URL=$SEQ_SERVER_URL - ./devops/dotenv --file .env set SEQ_API_KEY=$SEQ_API_KEY - ./devops/dotenv --file .env set SLACK_TOKEN=$SLACK_TOKEN - - # Calculate port offsets based on deployment slot - BASE_PORT=3000 - SLOT_MULTIPLIER=10 - PORT_OFFSET=$((BASE_PORT + DEPLOYMENT_SLOT_PORT_OFFSET * SLOT_MULTIPLIER)) - - # Set addresses using consistent port calculation - APP_BIND_ADDRESS="127.0.0.1:$PORT_OFFSET" - ./devops/dotenv --file .env set APP_BIND_ADDRESS=$APP_BIND_ADDRESS - - SUPABASE_BIND_ADDRESS="127.0.0.1:$((PORT_OFFSET + 1))" - ./devops/dotenv --file .env set SUPABASE_BIND_ADDRESS=$SUPABASE_BIND_ADDRESS - - DB_PUBLIC_LOCALHOST_PORT="$((PORT_OFFSET + 2))" - ./devops/dotenv --file .env set DB_PUBLIC_LOCALHOST_PORT=$DB_PUBLIC_LOCALHOST_PORT - - echo "Setting Supabase Container Configuration" - - ./devops/dotenv --file .env set POSTGRES_PASSWORD=$POSTGRES_PASSWORD - ./devops/dotenv --file .env set JWT_SECRET=$JWT_SECRET - ./devops/dotenv --file .env set DASHBOARD_USERNAME=$DASHBOARD_USERNAME - ./devops/dotenv --file .env set DASHBOARD_PASSWORD=$DASHBOARD_PASSWORD - - ./devops/dotenv --file .env set SITE_URL=$STATBUS_URL - ./devops/dotenv --file .env set API_EXTERNAL_URL=$BROWSER_SUPABASE_URL - ./devops/dotenv --file .env set SUPABASE_PUBLIC_URL=$BROWSER_SUPABASE_URL - # Maps to GOTRUE_EXTERNAL_EMAIL_ENABLED to allow authentication with Email at all. - # So SIGNUP really means SIGNIN - ./devops/dotenv --file .env set ENABLE_EMAIL_SIGNUP=true - # Allow creating users and setting the email as verified, - # rather than sending an actual email where the user must - # click the link. - ./devops/dotenv --file .env set ENABLE_EMAIL_AUTOCONFIRM=true - # Disables signup with EMAIL, when ENABLE_EMAIL_SIGNUP=true - ./devops/dotenv --file .env set DISABLE_SIGNUP=true - # Sets the project name in the Supabase API portal. - ./devops/dotenv --file .env set STUDIO_DEFAULT_PROJECT="$DEPLOYMENT_SLOT_NAME" - - # JWT Tokens, used by Supabase Docker images - ./devops/dotenv --file .env set ANON_KEY=$ANON_KEY - ./devops/dotenv --file .env set SERVICE_ROLE_KEY=$SERVICE_ROLE_KEY - - # Add Publicly exposed Next.js variables - cat >> .env < deployment.caddyfile <