diff --git a/.gitignore b/.gitignore index 5e297107..2b145fc6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ until_fail.sh # Secrets apps/*/priv/secrets/* !apps/*/priv/secrets/.keep +**/*.secret.exs # Elixir LS .elixir_ls diff --git a/.travis.yml b/.travis.yml index 5c46c86c..83858f57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,7 @@ -# ----- Config ----- - stages: - name: test - name: release -env: - CF_API_REST_IMAGE: captainfact/rest-api:$TRAVIS_BRANCH - CF_API_GRAPHQL_IMAGE: captainfact/graphql-api:$TRAVIS_BRANCH - CF_API_ATOM_FEED: captainfact/atom-feed:$TRAVIS_BRANCH - CF_API_OPENGRAPH_IMAGE: captainfact/opengraph:$TRAVIS_BRANCH - CF_API_JOBS_IMAGE: captainfact/jobs:$TRAVIS_BRANCH - # ---- Jobs ---- jobs: @@ -24,6 +15,7 @@ jobs: env: - MIX_ENV=test before_script: + - mix format --check-formatted - mix local.hex --force - mix local.rebar --force - mix deps.get @@ -31,7 +23,6 @@ jobs: - mix ecto.migrate script: - mix coveralls.travis --umbrella - - mix format --check-formatted - stage: release if: branch IN (master, staging) AND type != pull_request @@ -39,17 +30,4 @@ jobs: sudo: required services: [docker] script: - - set -e - - # ---- Build ---- - - docker build --build-arg APP=cf_rest_api -t ${CF_API_REST_IMAGE} . - - docker build --build-arg APP=cf_graphql -t ${CF_API_GRAPHQL_IMAGE} . - - docker build --build-arg APP=cf_atom_feed -t ${CF_API_ATOM_FEED} . - - docker build --build-arg APP=cf_opengraph -t ${CF_API_OPENGRAPH_IMAGE} . - - docker build --build-arg APP=cf_jobs -t ${CF_API_JOBS_IMAGE} . - - # ---- Push release ---- - - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"; - - docker push $CF_API_REST_IMAGE - - docker push $CF_API_GRAPHQL_IMAGE - - docker push $CF_API_ATOM_FEED - - docker push $CF_API_OPENGRAPH_IMAGE - - docker push $CF_API_JOBS_IMAGE + - ./rel/release.sh $TRAVIS_BRANCH diff --git a/README.md b/README.md index ea8e62cd..3ddef81d 100644 --- a/README.md +++ b/README.md @@ -16,27 +16,26 @@

-## Install & Run +# Install & Run -### Start DB +## Prerequisites + +You need to install Elixir. We recommand using [asdf](https://github.com/asdf-vm/asdf#setup). +Check their documentation on how to install it, then run `asdf install` from +root `captain-fact-api` folder. + +## Start DB Create / launch a postgres instance on your local machine. If you have docker installed, you can use the pre-seed postgres docker image: `docker run -d --name cf_dev_db -p 5432:5432 captainfact/dev-db:latest` -### Start API services - -- Without Docker (recommended if you want to make changes in the API) +## Start API - - `mix deps.get` - - `mix ecto.migrate` - - `iex -S mix` - -- With Docker - - Download project's dependencies with `./dev/get_dependencies.sh` - - Migrate your database with `./dev/db_migrate.sh` - - Start server with `./dev/start_server.sh` +- `mix deps.get` --> Get dependencies +- `mix ecto.migrate` --> Migrate DB +- `iex -S mix` --> Start project Following services will be started: @@ -46,16 +45,16 @@ Following services will be started: - [localhost:4003](https://localhost:4003) - GraphQL API (https) - [localhost:4004](http://localhost:4004) - Atom feed -You can run tests with `./dev/test.sh`. You can filter which tests to run by -running something like `./dev/test.sh test/your_test_subpath`. -Check `./dev/test.sh` script comments for details. +## Other useful commands -## Project architecture +- `mix test` --> Run tests +- `mix test.watch` --> Run tests watcher +- `mix format` --> Format code +- `mix ecto.gen.migration [migration_name]` --> Generate migration -Elixir offers very nice ways to separate concerns and work with microservices. -This application is organized as an [umbrella project](https://elixir-lang.org/getting-started/mix-otp/dependencies-and-umbrella-apps.html) which allows us to divide CaptainFact API into small apps. +# Project architecture -### File structure +This application is organized as an [umbrella project](https://elixir-lang.org/getting-started/mix-otp/dependencies-and-umbrella-apps.html) which allows us to divide CaptainFact API into small apps. ``` . @@ -82,21 +81,9 @@ This application is organized as an [umbrella project](https://elixir-lang.org/g │   └── config.exs => Releases configuration. ``` -## Styling - -Code should follow [Elixy Style Guide](https://github.com/christopheradams/elixir_style_guide) -and [Credo style guide](https://github.com/rrrene/elixir-style-guide) -as much as possible. - -Avoid lines longer than 80 characters, **never** go beyond 110 characters. - -## Linked projects +# Linked projects - [Community discussions and documentation](https://github.com/CaptainFact/captain-fact/) - [Frontend](https://github.com/CaptainFact/captain-fact-frontend) - [Extension](https://github.com/CaptainFact/captain-fact-extension) - [Overlay injector](https://github.com/CaptainFact/captain-fact-overlay-injector) - -# Feature requests - -[![Feature Requests](http://feathub.com/CaptainFact/captain-fact?format=svg)](http://feathub.com/CaptainFact/captain-fact) diff --git a/apps/cf/config/dev.exs b/apps/cf/config/dev.exs index 4ddf87ab..0f82cadb 100644 --- a/apps/cf/config/dev.exs +++ b/apps/cf/config/dev.exs @@ -31,3 +31,6 @@ config :rollbax, # Mails config :cf, CF.Mailer, adapter: Bamboo.LocalAdapter + +# Import local secrets if any - use wildcard to ignore errors +import_config "*dev.secret.exs" diff --git a/apps/cf/config/test.exs b/apps/cf/config/test.exs index 400276f9..dfd0f82f 100644 --- a/apps/cf/config/test.exs +++ b/apps/cf/config/test.exs @@ -23,9 +23,9 @@ config :cf, CF.Mailer, adapter: Bamboo.TestAdapter # Reduce the number of round for encryption during tests config :bcrypt_elixir, :log_rounds, 4 -# Captions mock for testing -config :cf, - captions_fetcher: CF.Videos.CaptionsFetcherTest +# Behaviours mock for testing +config :cf, captions_fetcher: CF.Videos.CaptionsFetcherTest +config :cf, use_test_video_metadata_fetcher: true # Configure Rollbar (errors reporting) config :rollbax, diff --git a/apps/cf/lib/accounts/accounts.ex b/apps/cf/lib/accounts/accounts.ex index 8ff4c878..56dee09d 100644 --- a/apps/cf/lib/accounts/accounts.ex +++ b/apps/cf/lib/accounts/accounts.ex @@ -48,11 +48,27 @@ defmodule CF.Accounts do # Do create user user_params + |> prepare_user_params_from_third_party() |> create_account_from_params(provider_params, allow_empty_username) |> after_create(invitation) end end + # Special formating for third-party provided user params + defp prepare_user_params_from_third_party(params) do + # Truncate name to avoid crashing when registering with a too-long name + cond do + Map.has_key?(params, :name) -> + Map.update(params, :name, nil, &String.slice(&1, 0..19)) + + Map.has_key?(params, "name") -> + Map.update(params, "name", nil, &String.slice(&1, 0..19)) + + true -> + params + end + end + defp create_account_from_params(user_params, provider_params, allow_empty_username) do case Map.get(user_params, "username") || Map.get(user_params, :username) do username when allow_empty_username and (is_nil(username) or username == "") -> diff --git a/apps/cf/lib/actions/action_creator.ex b/apps/cf/lib/actions/action_creator.ex index 182c2be3..0c5ee518 100644 --- a/apps/cf/lib/actions/action_creator.ex +++ b/apps/cf/lib/actions/action_creator.ex @@ -100,7 +100,7 @@ defmodule CF.Actions.ActionCreator do user_id, :video, :update, - video_id: video.video_id, + video_id: video.id, changes: changes ) end diff --git a/apps/cf/lib/videos/captions_fetcher.ex b/apps/cf/lib/videos/captions_fetcher.ex index 7ba7be4f..649d294f 100644 --- a/apps/cf/lib/videos/captions_fetcher.ex +++ b/apps/cf/lib/videos/captions_fetcher.ex @@ -3,6 +3,5 @@ defmodule CF.Videos.CaptionsFetcher do Fetch captions for videos. """ - @callback fetch(String.t(), String.t()) :: - {:ok, DB.Schema.VideoCaption.t()} | {:error, binary()} + @callback fetch(DB.Schema.Video.t()) :: {:ok, DB.Schema.VideoCaption.t()} | {:error, binary()} end diff --git a/apps/cf/lib/videos/captions_fetcher_test.ex b/apps/cf/lib/videos/captions_fetcher_test.ex index 9295049a..178e08ca 100644 --- a/apps/cf/lib/videos/captions_fetcher_test.ex +++ b/apps/cf/lib/videos/captions_fetcher_test.ex @@ -6,7 +6,7 @@ defmodule CF.Videos.CaptionsFetcherTest do @behaviour CF.Videos.CaptionsFetcher @impl true - def fetch(_provider_id, _locale) do + def fetch(_video) do captions = %DB.Schema.VideoCaption{ content: "__TEST-CONTENT__", format: "xml" diff --git a/apps/cf/lib/videos/captions_fetcher_youtube.ex b/apps/cf/lib/videos/captions_fetcher_youtube.ex index 4403924d..5081c689 100644 --- a/apps/cf/lib/videos/captions_fetcher_youtube.ex +++ b/apps/cf/lib/videos/captions_fetcher_youtube.ex @@ -6,7 +6,7 @@ defmodule CF.Videos.CaptionsFetcherYoutube do @behaviour CF.Videos.CaptionsFetcher @impl true - def fetch(youtube_id, locale) do + def fetch(%{youtube_id: youtube_id, locale: locale}) do with {:ok, content} <- fetch_captions_content(youtube_id, locale) do captions = %DB.Schema.VideoCaption{ content: content, diff --git a/apps/cf/lib/videos/metadata_fetcher.ex b/apps/cf/lib/videos/metadata_fetcher.ex index 96dda2e9..9df30716 100644 --- a/apps/cf/lib/videos/metadata_fetcher.ex +++ b/apps/cf/lib/videos/metadata_fetcher.ex @@ -1,76 +1,16 @@ defmodule CF.Videos.MetadataFetcher do @moduledoc """ - Methods to fetch metadata (title, language) from videos + Fetch metadata for video. """ - require Logger - - alias Kaur.Result - alias GoogleApi.YouTube.V3.Connection, as: YouTubeConnection - alias GoogleApi.YouTube.V3.Api.Videos, as: YouTubeVideos - alias GoogleApi.YouTube.V3.Model.Video, as: YouTubeVideo - alias GoogleApi.YouTube.V3.Model.VideoListResponse, as: YouTubeVideoList - - alias DB.Schema.Video + @type video_metadata :: %{ + title: String.t(), + language: String.t(), + url: String.t() + } @doc """ - Fetch metadata from video. Returns an object containing :title, :url and :language - - Usage: - iex> fetch_video_metadata("https://www.youtube.com/watch?v=OhWRT3PhMJs") - iex> fetch_video_metadata({"youtube", "OhWRT3PhMJs"}) + Takes an URL, fetch the metadata and return them """ - def fetch_video_metadata(nil), - do: {:error, "Invalid URL"} - - if Application.get_env(:db, :env) == :test do - def fetch_video_metadata(url = "__TEST__/" <> _id) do - {:ok, %{title: "__TEST-TITLE__", url: url}} - end - end - - def fetch_video_metadata(url) when is_binary(url), - do: fetch_video_metadata(Video.parse_url(url)) - - def fetch_video_metadata({"youtube", provider_id}) do - case Application.get_env(:cf, :youtube_api_key) do - nil -> - Logger.warn("No YouTube API key provided. Falling back to HTML fetcher") - fetch_video_metadata_html("youtube", provider_id) - - api_key -> - fetch_video_metadata_api("youtube", provider_id, api_key) - end - end - - defp fetch_video_metadata_api("youtube", provider_id, api_key) do - YouTubeConnection.new() - |> YouTubeVideos.youtube_videos_list("snippet", id: provider_id, key: api_key) - |> Result.map_error(fn e -> "YouTube API Error: #{inspect(e)}" end) - |> Result.keep_if(&(!Enum.empty?(&1.items)), "Video doesn't exist") - |> Result.map(fn %YouTubeVideoList{items: [video = %YouTubeVideo{} | _]} -> - %{ - title: video.snippet.title, - language: video.snippet.defaultLanguage || video.snippet.defaultAudioLanguage, - url: Video.build_url(%{provider: "youtube", provider_id: provider_id}) - } - end) - end - - defp fetch_video_metadata_html(provider, id) do - url = Video.build_url(%{provider: provider, provider_id: id}) - - case HTTPoison.get(url) do - {:ok, %HTTPoison.Response{body: body}} -> - meta = Floki.attribute(body, "meta[property='og:title']", "content") - - case meta do - [] -> {:error, "Page does not contains an OpenGraph title attribute"} - [title] -> {:ok, %{title: HtmlEntities.decode(title), url: url}} - end - - {_, _} -> - {:error, "Remote URL didn't respond correctly"} - end - end + @callback fetch_video_metadata(String.t()) :: {:ok, video_metadata} | {:error, binary()} end diff --git a/apps/cf/lib/videos/metadata_fetcher_opengraph.ex b/apps/cf/lib/videos/metadata_fetcher_opengraph.ex new file mode 100644 index 00000000..61de6ecd --- /dev/null +++ b/apps/cf/lib/videos/metadata_fetcher_opengraph.ex @@ -0,0 +1,25 @@ +defmodule CF.Videos.MetadataFetcher.Opengraph do + @moduledoc """ + Methods to fetch metadata (title, language) from videos + """ + + @behaviour CF.Videos.MetadataFetcher + + @doc """ + Fetch metadata from video using OpenGraph tags. + """ + def fetch_video_metadata(url) do + case HTTPoison.get(url) do + {:ok, %HTTPoison.Response{body: body}} -> + meta = Floki.attribute(body, "meta[property='og:title']", "content") + + case meta do + [] -> {:error, "Page does not contains an OpenGraph title attribute"} + [title] -> {:ok, %{title: HtmlEntities.decode(title), url: url}} + end + + {_, _} -> + {:error, "Remote URL didn't respond correctly"} + end + end +end diff --git a/apps/cf/lib/videos/metadata_fetcher_test.ex b/apps/cf/lib/videos/metadata_fetcher_test.ex new file mode 100644 index 00000000..948767cf --- /dev/null +++ b/apps/cf/lib/videos/metadata_fetcher_test.ex @@ -0,0 +1,18 @@ +defmodule CF.Videos.MetadataFetcher.Test do + @moduledoc """ + Methods to fetch metadata (title, language) from videos + """ + + @behaviour CF.Videos.MetadataFetcher + + @doc """ + Fetch metadata from video using OpenGraph tags. + """ + def fetch_video_metadata(url) do + {:ok, + %{ + title: "__TEST-TITLE__", + url: url + }} + end +end diff --git a/apps/cf/lib/videos/metadata_fetcher_youtube.ex b/apps/cf/lib/videos/metadata_fetcher_youtube.ex new file mode 100644 index 00000000..0a20ae01 --- /dev/null +++ b/apps/cf/lib/videos/metadata_fetcher_youtube.ex @@ -0,0 +1,51 @@ +defmodule CF.Videos.MetadataFetcher.Youtube do + @moduledoc """ + Methods to fetch metadata (title, language) from videos + """ + + @behaviour CF.Videos.MetadataFetcher + + require Logger + + alias Kaur.Result + alias GoogleApi.YouTube.V3.Connection, as: YouTubeConnection + alias GoogleApi.YouTube.V3.Api.Videos, as: YouTubeVideos + alias GoogleApi.YouTube.V3.Model.Video, as: YouTubeVideo + alias GoogleApi.YouTube.V3.Model.VideoListResponse, as: YouTubeVideoList + + alias DB.Schema.Video + alias CF.Videos.MetadataFetcher + + @doc """ + Fetch metadata from video. Returns an object containing :title, :url and :language + """ + def fetch_video_metadata(nil), + do: {:error, "Invalid URL"} + + def fetch_video_metadata(url) when is_binary(url) do + {:youtube, youtube_id} = Video.parse_url(url) + + case Application.get_env(:cf, :youtube_api_key) do + nil -> + Logger.warn("No YouTube API key provided. Falling back to HTML fetcher") + MetadataFetcher.Opengraph.fetch_video_metadata(url) + + api_key -> + do_fetch_video_metadata(youtube_id, api_key) + end + end + + defp do_fetch_video_metadata(youtube_id, api_key) do + YouTubeConnection.new() + |> YouTubeVideos.youtube_videos_list("snippet", id: youtube_id, key: api_key) + |> Result.map_error(fn e -> "YouTube API Error: #{inspect(e)}" end) + |> Result.keep_if(&(!Enum.empty?(&1.items)), "remote_video_404") + |> Result.map(fn %YouTubeVideoList{items: [video = %YouTubeVideo{} | _]} -> + %{ + title: video.snippet.title, + language: video.snippet.defaultLanguage || video.snippet.defaultAudioLanguage, + url: Video.build_url(%{youtube_id: youtube_id}) + } + end) + end +end diff --git a/apps/cf/lib/videos/videos.ex b/apps/cf/lib/videos/videos.ex index 1facbbf2..e3129af1 100644 --- a/apps/cf/lib/videos/videos.ex +++ b/apps/cf/lib/videos/videos.ex @@ -6,17 +6,18 @@ defmodule CF.Videos do import Ecto.Query, warn: false import CF.Videos.MetadataFetcher import CF.Videos.CaptionsFetcher + import CF.Actions.ActionCreator, only: [action_update: 2] alias Ecto.Multi alias DB.Repo alias DB.Schema.Video - alias DB.Schema.Statement alias DB.Schema.Speaker alias DB.Schema.VideoSpeaker alias DB.Schema.VideoCaption alias CF.Actions.ActionCreator alias CF.Accounts.UserPermissions + alias CF.Videos.MetadataFetcher @captions_fetcher Application.get_env(:cf, :captions_fetcher) @@ -34,28 +35,17 @@ defmodule CF.Videos do def videos_list(filters, false), do: Repo.all(Video.query_list(Video, filters)) - @doc """ - Index videos, returning only their id, provider_id and provider. - Accepted filters are the same than for `videos_list/1` - """ - def videos_index(from_id \\ 0) do - Video - |> select([v], %{id: v.id, provider: v.provider, provider_id: v.provider_id}) - |> where([v], v.id > ^from_id) - |> Repo.all() - end - @doc """ Return the corresponding video if it has already been added, `nil` otherwise """ def get_video_by_url(url) do case Video.parse_url(url) do - {provider, id} -> + {:youtube, id} -> Video |> Video.with_speakers() - |> Repo.get_by(provider: provider, provider_id: id) + |> Repo.get_by(youtube_id: id) - nil -> + _ -> nil end end @@ -74,7 +64,8 @@ defmodule CF.Videos do def create!(user, video_url, is_partner \\ nil) do UserPermissions.check!(user, :add, :video) - with {:ok, metadata} <- fetch_video_metadata(video_url) do + with metadata_fetcher when not is_nil(metadata_fetcher) <- get_metadata_fetcher(video_url), + {:ok, metadata} <- metadata_fetcher.(video_url) do # Videos posted by publishers are recorded as partner unless explicitely # specified otherwise (false) base_video = %Video{is_partner: user.is_publisher && is_partner != false} @@ -106,33 +97,20 @@ defmodule CF.Videos do Returns {:ok, statements} if success, {:error, reason} otherwise. Returned statements contains only an id and a key """ - def shift_statements(user, video_id, offset) when is_integer(offset) do + def shift_statements(user, video_id, offsets) do UserPermissions.check!(user, :update, :video) - statements_query = where(Statement, [s], s.video_id == ^video_id) + video = Repo.get!(Video, video_id) + changeset = Video.changeset_shift_offsets(video, offsets) Multi.new() - |> Multi.update_all( - :statements_update, - statements_query, - [inc: [time: offset]], - returning: [:id, :time] - ) - |> Multi.insert( - :action, - ActionCreator.action( - user.id, - :video, - :update, - video_id: video_id, - changes: %{"statements_time" => offset} - ) - ) + |> Multi.update(:video, changeset) + |> Multi.insert(:action_update, action_update(user.id, changeset)) |> Repo.transaction() |> case do - {:ok, %{statements_update: {_, statements}}} -> - {:ok, Enum.map(statements, &%{id: &1.id, time: &1.time})} + {:ok, %{video: video}} -> + {:ok, video} - {:error, _, reason, _} -> + {:error, _operation, reason, _changes} -> {:error, reason} end end @@ -163,10 +141,9 @@ defmodule CF.Videos do Usage: iex> download_captions(video) - iex> download_captions(video) """ def download_captions(video = %Video{}) do - with {:ok, captions} <- @captions_fetcher.fetch(video.provider_id, video.language) do + with {:ok, captions} <- @captions_fetcher.fetch(video) do captions |> VideoCaption.changeset(%{video_id: video.id}) |> Repo.insert() @@ -174,4 +151,20 @@ defmodule CF.Videos do {:ok, captions} end end + + defp get_metadata_fetcher(video_url) do + cond do + Application.get_env(:cf, :use_test_video_metadata_fetcher) -> + &MetadataFetcher.Test.fetch_video_metadata/1 + + # We only support YouTube for now + # TODO Use a Regex here + video_url -> + &MetadataFetcher.Youtube.fetch_video_metadata/1 + + # Use a default fetcher that retrieves info from OpenGraph tags + true -> + &MetadataFetcher.Opengraph.fetch_video_metadata/1 + end + end end diff --git a/apps/cf/mix.exs b/apps/cf/mix.exs index 221f525c..b9ef4810 100644 --- a/apps/cf/mix.exs +++ b/apps/cf/mix.exs @@ -4,7 +4,7 @@ defmodule CF.Mixfile do def project do [ app: :cf, - version: "0.9.0", + version: "0.9.1", build_path: "../../_build", compilers: [:phoenix, :gettext] ++ Mix.compilers(), config_path: "../../config/config.exs", diff --git a/apps/cf/priv/secrets/.keep b/apps/cf/priv/secrets/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/cf/test/accounts/accounts_test.exs b/apps/cf/test/accounts/accounts_test.exs index ecf35f5f..a333dc88 100644 --- a/apps/cf/test/accounts/accounts_test.exs +++ b/apps/cf/test/accounts/accounts_test.exs @@ -162,6 +162,13 @@ defmodule CF.AccountsTest do Accounts.create_account(user_params, invit.token, provider_params: provider_params) end + test "truncate name if too long" do + user_params = Map.put(build_user_params(), :name, "abcdefghijklmnopqrstuvwxyz") + provider_params = %{fb_user_id: "4242424242"} + {:ok, user} = Accounts.create_account(user_params, nil, provider_params: provider_params) + assert user.name == "abcdefghijklmnopqrst" + end + test "delete invitation request after creating the user" do Repo.delete_all(DB.Schema.InvitationRequest) invit = insert(:invitation_request) diff --git a/apps/cf/test/videos/videos_test.exs b/apps/cf/test/videos/videos_test.exs index 6efde826..e70cefa9 100644 --- a/apps/cf/test/videos/videos_test.exs +++ b/apps/cf/test/videos/videos_test.exs @@ -5,7 +5,7 @@ defmodule CF.VideosTest do alias CF.Videos alias CF.Accounts.UserPermissions.PermissionsError - defp test_url, do: "__TEST__/#{DB.Utils.TokenGenerator.generate(8)}" + defp test_url, do: "https://www.youtube.com/watch?v=#{DB.Utils.TokenGenerator.generate(11)}" describe "Add video" do test "without enough reputation" do @@ -59,7 +59,13 @@ defmodule CF.VideosTest do describe "Fetch captions" do test "fetch captions" do - video = DB.Factory.insert(:video, provider: "__TEST__", language: "en") + video = + DB.Factory.insert( + :video, + youtube_id: DB.Utils.TokenGenerator.generate(11), + language: "en" + ) + {:ok, captions} = Videos.download_captions(video) assert captions.content == "__TEST-CONTENT__" diff --git a/apps/cf_atom_feed/lib/flags.ex b/apps/cf_atom_feed/lib/flags.ex new file mode 100644 index 00000000..deb2d05b --- /dev/null +++ b/apps/cf_atom_feed/lib/flags.ex @@ -0,0 +1,90 @@ +defmodule CF.AtomFeed.Flags do + @moduledoc """ + Generate an ATOM feed that contains all flags + """ + + import Ecto.Query + + alias Atomex.{Feed, Entry} + alias DB.Schema.{Flag, UserAction, User, Comment} + alias DB.Type.FlagReason + alias CF.Utils.FrontendRouter + + @nb_items_max 50 + + @doc """ + Get an ATOM feed containing all site's flags in reverse chronological + order (newest first) + """ + def feed_all() do + flags = fetch_flags() + generate_feed(flags, last_update(flags)) + end + + defp fetch_flags() do + DB.Repo.all( + from( + flag in Flag, + order_by: [desc: flag.inserted_at], + left_join: action in assoc(flag, :action), + left_join: comment in assoc(action, :comment), + left_join: user in assoc(comment, :user), + left_join: source in assoc(comment, :source), + preload: [action: [comment: [:user, :statement, :source]]], + limit: @nb_items_max + ) + ) + end + + defp last_update(_flags = [flag | _]), + do: DateTime.from_naive!(flag.inserted_at, "Etc/UTC") + + defp last_update(_), + do: DateTime.utc_now() + + defp generate_feed(flags, last_update) do + FrontendRouter.base_url() + |> Feed.new(last_update, "[CaptainFact] All Flags") + |> CF.AtomFeed.Common.feed_author() + |> Feed.link("https://feed.captainfact.io/flags/", rel: "self") + |> Feed.entries(Enum.map(flags, &get_entry/1)) + |> Feed.build() + |> Atomex.generate_document() + end + + defp get_entry(flag) do + comment = flag.action.comment + user_appelation = User.user_appelation(comment.user) + title = entry_title(flag, user_appelation) + link = comment_url(flag.action) + insert_datetime = DateTime.from_naive!(flag.inserted_at, "Etc/UTC") + + link + |> Entry.new(insert_datetime, title) + |> Entry.link(link) + |> Entry.published(insert_datetime) + |> Entry.content(""" + ``` + #{comment.text} + ``` + Source Comment: #{source(comment)}\n + Flag reason: #{FlagReason.label(flag.reason)} + """) + |> Entry.build() + end + + defp source(%{source: nil}), + do: "None" + + defp source(%{source: %{url: url, site_name: site_name}}), + do: "[#{site_name || url}](#{url})" + + defp entry_title(flag, user_appelation) do + "New Flag for #{user_appelation} comment ##{flag.action.comment.id}" + end + + defp comment_url(action) do + video_hash_id = DB.Type.VideoHashId.encode(action.video_id) + FrontendRouter.comment_url(video_hash_id, action.comment) + end +end diff --git a/apps/cf_atom_feed/lib/router.ex b/apps/cf_atom_feed/lib/router.ex index 771a931a..40bd8c2b 100644 --- a/apps/cf_atom_feed/lib/router.ex +++ b/apps/cf_atom_feed/lib/router.ex @@ -42,4 +42,8 @@ defmodule CF.AtomFeed.Router do get "/videos" do render_feed(conn, CF.AtomFeed.Videos.feed_all()) end + + get "/flags" do + render_feed(conn, CF.AtomFeed.Flags.feed_all()) + end end diff --git a/apps/cf_atom_feed/lib/videos.ex b/apps/cf_atom_feed/lib/videos.ex index f45dbb3a..18bdfe22 100644 --- a/apps/cf_atom_feed/lib/videos.ex +++ b/apps/cf_atom_feed/lib/videos.ex @@ -40,30 +40,28 @@ defmodule CF.AtomFeed.Videos do defp render_entry(video) do video_link = FrontendRouter.video_url(video.hash_id) - insert_datetime = video_datetime(video) video_link - |> Entry.new(insert_datetime, video.title) + |> Entry.new(video.inserted_at, video.title) |> Entry.link(video_link) - |> Entry.published(insert_datetime) + |> Entry.published(video.inserted_at) |> Entry.content(entry_content(video)) |> Entry.build() end + defp entry_content(%{speakers: []}) do + "" + end + defp entry_content(video) do - """ - Speakers: #{Enum.map_join(video.speakers, ", ", &render_speaker/1)} - """ + Enum.map_join(video.speakers, ", ", &render_speaker/1) end defp render_speaker(speaker = %{full_name: name}), do: "[#{name}](#{FrontendRouter.speaker_url(speaker)})" - defp video_datetime(%{inserted_at: naive_datetime}), - do: DateTime.from_naive!(naive_datetime, "Etc/UTC") - defp last_update([video | _]), - do: video_datetime(video) + do: video.inserted_at defp last_update(_), do: DateTime.utc_now() diff --git a/apps/cf_atom_feed/mix.exs b/apps/cf_atom_feed/mix.exs index 9fbeb790..daf6d507 100644 --- a/apps/cf_atom_feed/mix.exs +++ b/apps/cf_atom_feed/mix.exs @@ -4,7 +4,7 @@ defmodule CF.AtomFeed.Mixfile do def project do [ app: :cf_atom_feed, - version: "0.9.0", + version: "0.9.1", build_path: "../../_build", config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/apps/cf_graphql/lib/resolvers/videos.ex b/apps/cf_graphql/lib/resolvers/videos.ex index a3dd658c..e60bd209 100644 --- a/apps/cf_graphql/lib/resolvers/videos.ex +++ b/apps/cf_graphql/lib/resolvers/videos.ex @@ -15,21 +15,21 @@ defmodule CF.GraphQL.Resolvers.Videos do # Queries def get(_root, %{id: id}, _info) do - case get_video_by_id(id) do + case CF.Videos.get_video_by_id(id) do nil -> {:error, "Video #{id} doesn't exist"} video -> {:ok, video} end end def get(_root, %{hash_id: id}, _info) do - case get_video_by_id(id) do + case CF.Videos.get_video_by_id(id) do nil -> {:error, "Video #{id} doesn't exist"} video -> {:ok, video} end end def get(_root, %{url: url}, _info) do - case get_video_by_url(url) do + case CF.Videos.get_video_by_url(url) do nil -> {:error, "Video with url #{url} doesn't exist"} video -> {:ok, video} end @@ -69,20 +69,4 @@ defmodule CF.GraphQL.Resolvers.Videos do |> Repo.all() |> Enum.group_by(& &1.video_id) end - - # ---- Private ---- - - defp get_video_by_url(url) do - case Video.parse_url(url) do - {provider, id} -> - Video - |> Video.with_speakers() - |> Repo.get_by(provider: provider, provider_id: id) - - nil -> - nil - end - end - - defp get_video_by_id(id), do: Repo.get(Video, id) end diff --git a/apps/cf_graphql/lib/schema/types/video.ex b/apps/cf_graphql/lib/schema/types/video.ex index ff7698a9..e104e889 100644 --- a/apps/cf_graphql/lib/schema/types/video.ex +++ b/apps/cf_graphql/lib/schema/types/video.ex @@ -21,9 +21,15 @@ defmodule CF.GraphQL.Schema.Types.Video do @desc "Video URL" field(:url, non_null(:string), do: resolve(&Resolvers.Videos.url/3)) @desc "Video provider (youtube, vimeo...etc)" - field(:provider, non_null(:string)) + field(:provider, non_null(:string), deprecate: "Use `url` or `youtube_id`") do + resolve(fn _, _, _ -> {:ok, "youtube"} end) + end + @desc "Unique ID used to identify video with provider" - field(:provider_id, non_null(:string)) + field(:provider_id, non_null(:string), deprecate: "Use `url` or `youtube_id`") do + resolve(fn v, _, _ -> {:ok, v.youtube_id} end) + end + @desc "Language of the video represented as a two letters locale" field(:language, :string) @desc "Video insert datetime" @@ -41,6 +47,14 @@ defmodule CF.GraphQL.Schema.Types.Video do resolve(&Resolvers.Videos.statements/3) complexity(join_complexity()) end + + # Video providers + + @desc "Youtube ID for this video" + field(:youtube_id, :string) + + @desc "Offset for all statements on this video when watched with YouTube player" + field(:youtube_offset, non_null(:integer)) end @desc "A list a paginated videos" diff --git a/apps/cf_graphql/mix.exs b/apps/cf_graphql/mix.exs index 1257023b..319c9490 100644 --- a/apps/cf_graphql/mix.exs +++ b/apps/cf_graphql/mix.exs @@ -4,7 +4,7 @@ defmodule CF.GraphQL.Mixfile do def project do [ app: :cf_graphql, - version: "0.9.0", + version: "0.9.1", build_path: "../../_build", config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/apps/cf_jobs/config/test.exs b/apps/cf_jobs/config/test.exs index d33b433c..5305a6d5 100644 --- a/apps/cf_jobs/config/test.exs +++ b/apps/cf_jobs/config/test.exs @@ -1,4 +1,4 @@ use Mix.Config # Disable CRON tasks on test -config :cf_jobs, CF.Scheduler, jobs: [] +config :cf_jobs, CF.Jobs.Scheduler, jobs: [] diff --git a/apps/cf_jobs/mix.exs b/apps/cf_jobs/mix.exs index 4e8e2fd9..eba4950b 100644 --- a/apps/cf_jobs/mix.exs +++ b/apps/cf_jobs/mix.exs @@ -4,7 +4,7 @@ defmodule CF.Jobs.Mixfile do def project do [ app: :cf_jobs, - version: "0.9.0", + version: "0.9.1", build_path: "../../_build", config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/apps/cf_opengraph/mix.exs b/apps/cf_opengraph/mix.exs index 3d2cc9fe..7ae53f2a 100644 --- a/apps/cf_opengraph/mix.exs +++ b/apps/cf_opengraph/mix.exs @@ -4,7 +4,7 @@ defmodule CF.Opengraph.MixProject do def project do [ app: :cf_opengraph, - version: "0.9.0", + version: "0.9.1", elixir: "~> 1.6", start_permanent: Mix.env() == :prod, deps: deps(), diff --git a/apps/cf_rest_api/lib/channels/statements_channel.ex b/apps/cf_rest_api/lib/channels/statements_channel.ex index cb7dd1e7..94b5850a 100644 --- a/apps/cf_rest_api/lib/channels/statements_channel.ex +++ b/apps/cf_rest_api/lib/channels/statements_channel.ex @@ -10,7 +10,6 @@ defmodule CF.RestApi.StatementsChannel do alias DB.Schema.Statement alias CF.Statements - alias CF.Videos alias CF.Accounts.UserPermissions alias CF.RestApi.{StatementView, ErrorView} @@ -32,26 +31,6 @@ defmodule CF.RestApi.StatementsChannel do handle_in_authenticated(command, params, socket, &handle_in_authenticated!/3) end - @doc """ - Shift all video's statements - """ - def handle_in_authenticated!("shift_all", offset, socket) do - user = Repo.get(DB.Schema.User, socket.assigns.user_id) - - case Videos.shift_statements( - user, - socket.assigns.video_id, - String.to_integer(offset) - ) do - {:ok, statements} -> - broadcast!(socket, "statements_updated", %{statements: statements}) - {:reply, :ok, socket} - - {:error, _} -> - {:reply, :error, socket} - end - end - @doc """ Add a new statement """ diff --git a/apps/cf_rest_api/lib/channels/user_socket.ex b/apps/cf_rest_api/lib/channels/user_socket.ex index 718719f6..27f38e34 100644 --- a/apps/cf_rest_api/lib/channels/user_socket.ex +++ b/apps/cf_rest_api/lib/channels/user_socket.ex @@ -16,6 +16,7 @@ defmodule CF.RestApi.UserSocket do ## Transports transport(:websocket, Phoenix.Transports.WebSocket) + transport(:longpoll, Phoenix.Transports.LongPoll) # Connect with token def connect(%{"token" => token}, socket) do diff --git a/apps/cf_rest_api/lib/channels/video_debate_channel.ex b/apps/cf_rest_api/lib/channels/video_debate_channel.ex index ba47e116..fd63a65c 100644 --- a/apps/cf_rest_api/lib/channels/video_debate_channel.ex +++ b/apps/cf_rest_api/lib/channels/video_debate_channel.ex @@ -18,6 +18,7 @@ defmodule CF.RestApi.VideoDebateChannel do alias DB.Schema.Speaker alias DB.Schema.VideoSpeaker + alias CF.Videos alias CF.Accounts.UserPermissions alias CF.RestApi.{VideoView, SpeakerView, ChangesetView} @@ -58,6 +59,27 @@ defmodule CF.RestApi.VideoDebateChannel do handle_in_authenticated(command, params, socket, &handle_in_authenticated!/3) end + @doc """ + Shift all video's statements + """ + def handle_in_authenticated!("shift_statements", offsets, socket) do + user = Repo.get(DB.Schema.User, socket.assigns.user_id) + + case Videos.shift_statements(user, socket.assigns.video_id, offsets) do + {:ok, video} -> + rendered_video = + video + |> DB.Repo.preload(:speakers) + |> View.render_one(VideoView, "video.json") + + broadcast!(socket, "video_updated", %{video: rendered_video}) + {:reply, :ok, socket} + + {:error, _} -> + {:reply, :error, socket} + end + end + @doc """ Add an existing speaker to the video """ diff --git a/apps/cf_rest_api/lib/controllers/video_controller.ex b/apps/cf_rest_api/lib/controllers/video_controller.ex index 690fb94a..21895037 100644 --- a/apps/cf_rest_api/lib/controllers/video_controller.ex +++ b/apps/cf_rest_api/lib/controllers/video_controller.ex @@ -5,6 +5,7 @@ defmodule CF.RestApi.VideoController do use CF.RestApi, :controller import CF.Videos + alias CF.RestApi.ChangesetView action_fallback(CF.RestApi.FallbackController) @@ -23,16 +24,6 @@ defmodule CF.RestApi.VideoController do def index(conn, _params), do: render(conn, :index, videos: videos_list()) - @doc """ - List all videos but only return indexes. - """ - @deprecated "Extension/Injector is now using GraphQL API for this" - def index_ids(conn, %{"min_id" => min_id}), - do: json(conn, videos_index(min_id)) - - def index_ids(conn, _), - do: json(conn, videos_index()) - @doc """ Create a new video based on `url`. If it already exist, just returns the video. @@ -44,10 +35,15 @@ defmodule CF.RestApi.VideoController do |> Guardian.Plug.current_resource() |> create!(url, params["is_partner"]) |> case do - {:error, message} -> + {:error, error} when is_binary(error) -> + conn + |> put_status(:unprocessable_entity) + |> json(%{error: error}) + + {:error, changeset = %Ecto.Changeset{}} -> conn |> put_status(:unprocessable_entity) - |> json(%{error: %{url: message}}) + |> json(ChangesetView.render("error.json", %{changeset: changeset})) {:ok, video} -> render(conn, "show.json", video: video) diff --git a/apps/cf_rest_api/lib/router.ex b/apps/cf_rest_api/lib/router.ex index dda93eee..cba25be3 100644 --- a/apps/cf_rest_api/lib/router.ex +++ b/apps/cf_rest_api/lib/router.ex @@ -24,7 +24,6 @@ defmodule CF.RestApi.Router do # ---- Public endpoints ---- get("/", ApiInfoController, :get) get("/videos", VideoController, :index) - get("/videos/index", VideoController, :index_ids) get("/speakers/:slug_or_id", SpeakerController, :show) post("/search/video", VideoController, :search) get("/videos/:video_id/statements", StatementController, :get) diff --git a/apps/cf_rest_api/lib/views/video_view.ex b/apps/cf_rest_api/lib/views/video_view.ex index 5d21685f..89bb5c66 100644 --- a/apps/cf_rest_api/lib/views/video_view.ex +++ b/apps/cf_rest_api/lib/views/video_view.ex @@ -14,8 +14,10 @@ defmodule CF.RestApi.VideoView do id: video.id, hash_id: video.hash_id, title: video.title, - provider: video.provider, - provider_id: video.provider_id, + provider: "youtube", + provider_id: video.youtube_id, + youtube_id: video.youtube_id, + youtube_offset: video.youtube_offset, url: DB.Schema.Video.build_url(video), posted_at: video.inserted_at, speakers: render_many(video.speakers, CF.RestApi.SpeakerView, "speaker.json"), diff --git a/apps/cf_rest_api/mix.exs b/apps/cf_rest_api/mix.exs index cd3d27eb..4cbf4653 100644 --- a/apps/cf_rest_api/mix.exs +++ b/apps/cf_rest_api/mix.exs @@ -4,7 +4,7 @@ defmodule CF.RestApi.Mixfile do def project do [ app: :cf_rest_api, - version: "0.9.0", + version: "0.9.1", build_path: "../../_build", compilers: [:phoenix, :gettext] ++ Mix.compilers(), config_path: "../../config/config.exs", diff --git a/apps/cf_rest_api/test/controllers/video_controller_test.exs b/apps/cf_rest_api/test/controllers/video_controller_test.exs index 1198f2b1..ce3a9efd 100644 --- a/apps/cf_rest_api/test/controllers/video_controller_test.exs +++ b/apps/cf_rest_api/test/controllers/video_controller_test.exs @@ -18,7 +18,7 @@ defmodule CF.RestApi.VideoControllerTest do assert Enum.count(videos) == Enum.count(response) assert random_video.title == - Enum.find(response, &(&1["provider_id"] == random_video.provider_id))["title"] + Enum.find(response, &(&1["youtube_id"] == random_video.youtube_id))["title"] end test "filter on language", %{conn: conn} do @@ -67,6 +67,6 @@ defmodule CF.RestApi.VideoControllerTest do |> post("/videos", %{url: "https://google.fr"}) |> json_response(422) - assert response == %{"error" => %{"url" => "Invalid URL"}} + assert response == %{"errors" => %{"url" => ["invalid_url"]}} end end diff --git a/apps/db/lib/db_schema/video.ex b/apps/db/lib/db_schema/video.ex index 1fad2c37..2ba5f4e7 100644 --- a/apps/db/lib/db_schema/video.ex +++ b/apps/db/lib/db_schema/video.ex @@ -13,31 +13,31 @@ defmodule DB.Schema.Video do field(:title, :string) field(:hash_id, :string) field(:url, :string, virtual: true) - field(:provider, :string, null: false) - field(:provider_id, :string, null: false) field(:language, :string, null: true) field(:unlisted, :boolean, null: false) field(:is_partner, :boolean, null: false) + # DEPRECATED + # field(:provider, :string) + # field(:provider_id, :string) + + field(:youtube_id, :string) + field(:youtube_offset, :integer, null: false, default: 0) + many_to_many(:speakers, Speaker, join_through: VideoSpeaker, on_delete: :delete_all) has_many(:statements, Statement, on_delete: :delete_all) - timestamps() + timestamps(type: :utc_datetime) end # Define valid providers @providers_regexs %{ - # Map a provider name to its regex, using named_captures to get the id --------------------↘️ - "youtube" => + # Map a provider name to its regex, using named_captures to get the id ---------↘️ + youtube: ~r/(?:youtube\.com\/(?:[^\/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)(?[^"&?\/ ]{11})/i } - # Allow for URLs like `https://anything.com/__TEST__/ID` in tests - if Application.get_env(:db, :env) == :test do - @providers_regexs Map.put(@providers_regexs, "__TEST__", ~r/__TEST__\/(?[^"&?\/ ]+)/) - end - # Define queries @doc """ @@ -108,28 +108,23 @@ defmodule DB.Schema.Video do ## Examples iex> import DB.Schema.Video, only: [build_url: 1] - iex> build_url(%{provider: "youtube", provider_id: "dQw4w9WgXcQ"}) + iex> build_url(%{youtube_id: "dQw4w9WgXcQ"}) "https://www.youtube.com/watch?v=dQw4w9WgXcQ" """ - def build_url(%{provider: "youtube", provider_id: id}), + def build_url(%{youtube_id: id}) when not is_nil(id), do: "https://www.youtube.com/watch?v=#{id}" # Add a special case for building test URLs if Application.get_env(:db, :env) == :test do - def build_url(%{provider: "__TEST__", provider_id: id}), + def build_url(%{youtube_id: id}), do: "__TEST__/#{id}" end @doc """ Returns overview image url for the given video """ - def image_url( - _video = %__MODULE__{ - provider: "youtube", - provider_id: youtube_id - } - ) do - "https://img.youtube.com/vi/#{youtube_id}/hqdefault.jpg" + def image_url(_video = %__MODULE__{youtube_id: id}) when not is_nil(id) do + "https://img.youtube.com/vi/#{id}/hqdefault.jpg" end @doc """ @@ -139,11 +134,10 @@ defmodule DB.Schema.Video do struct |> cast(params, [:url, :title, :language]) |> validate_required([:url, :title]) - |> parse_url() - |> validate_required([:provider, :provider_id]) + |> parse_video_url() |> validate_length(:title, min: 5, max: 120) - |> unique_constraint(:videos_provider_provider_id_index) - # Change "en-US" to "en" + |> unique_constraint(:videos_youtube_id_index) + # Change locales like "en-US" to "en" |> update_change(:language, &hd(String.split(&1, "-"))) end @@ -160,29 +154,26 @@ defmodule DB.Schema.Video do end @doc """ - Parse an URL. - If given a changeset, fill the `provider` and `provider_id` fields or add - an error if URL is not valid. - If given a binary, return {provider, id} or nil if invalid. + Builds a changeset that allows shifting statements for all providers ## Examples - iex> import DB.Schema.Video, only: [parse_url: 1] - iex> parse_url "" - nil - iex> parse_url "https://www.youtube.com/watch?v=dQw4w9WgXcQ" - {"youtube", "dQw4w9WgXcQ"} - iex> parse_url "https://www.youtube.com/watch?v=" - nil + iex> DB.Schema.Video.changeset_shift_offsets(%DB.Schema.Video{}, %{youtube_offset: 42}) + #Ecto.Changeset, valid?: true> + """ + def changeset_shift_offsets(struct, params \\ %{}) do + cast(struct, params, [:youtube_offset]) + end + + @doc """ + Given a changeset, fill the `{provider}_id` fields or add an error if URL is not valid. """ - def parse_url(changeset = %Ecto.Changeset{}) do + def parse_video_url(changeset = %Ecto.Changeset{}) do case changeset do %Ecto.Changeset{valid?: true, changes: %{url: url}} -> case parse_url(url) do - {provider, id} -> - changeset - |> put_change(:provider, provider) - |> put_change(:provider_id, id) + {:youtube, id} -> + put_change(changeset, :youtube_id, id) _ -> add_error(changeset, :url, "invalid_url") @@ -193,6 +184,20 @@ defmodule DB.Schema.Video do end end + @doc """ + Parse an URL. + Given a binary, return {provider, id} or nil if invalid. + + ## Examples + + iex> import DB.Schema.Video, only: [parse_url: 1] + iex> parse_url "" + nil + iex> parse_url "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + {:youtube, "dQw4w9WgXcQ"} + iex> parse_url "https://www.youtube.com/watch?v=" + nil + """ def parse_url(url) when is_binary(url) do Enum.find_value(@providers_regexs, fn {provider, regex} -> case Regex.named_captures(regex, url) do diff --git a/apps/db/lib/db_utils/string.ex b/apps/db/lib/db_utils/string.ex index 9ef2e313..3dbc7452 100644 --- a/apps/db/lib/db_utils/string.ex +++ b/apps/db/lib/db_utils/string.ex @@ -13,6 +13,12 @@ defmodule DB.Utils.String do iex> DB.Utils.String.trim_all_whitespaces "" "" """ - def trim_all_whitespaces(str), - do: String.replace(String.trim(str), ~r/\s+/, " ") + def trim_all_whitespaces(nil), + do: nil + + def trim_all_whitespaces(str) do + str + |> String.trim() + |> String.replace(~r/\s+/, " ") + end end diff --git a/apps/db/mix.exs b/apps/db/mix.exs index 785944e6..5b39ab35 100644 --- a/apps/db/mix.exs +++ b/apps/db/mix.exs @@ -4,7 +4,7 @@ defmodule DB.Mixfile do def project do [ app: :db, - version: "0.9.0", + version: "0.9.1", build_path: "../../_build", config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/apps/db/priv/repo/migrations/20180827123706_add_hash_id_to_videos.exs b/apps/db/priv/repo/migrations/20180827123706_add_hash_id_to_videos.exs index 26b95b4b..781f70a7 100644 --- a/apps/db/priv/repo/migrations/20180827123706_add_hash_id_to_videos.exs +++ b/apps/db/priv/repo/migrations/20180827123706_add_hash_id_to_videos.exs @@ -1,28 +1,31 @@ defmodule DB.Repo.Migrations.AddHashIdToVideos do use Ecto.Migration + import Ecto.Query alias DB.Schema.Video def up do alter table(:videos) do # A size of 10 allows us to go up to 100_000_000_000_000 videos - add :hash_id, :string, size: 10 + add(:hash_id, :string, size: 10) end # Create unique index on hash_id - create unique_index(:videos, [:hash_id]) + create(unique_index(:videos, [:hash_id])) # Flush pending migrations to ensure column is created flush() # Update all existing videos with their hashIds - DB.Repo.all(Video) + Video + |> select([v], v.id) + |> DB.Repo.all() |> Enum.map(&Video.changeset_generate_hash_id/1) |> Enum.map(&DB.Repo.update/1) end def down do alter table(:videos) do - remove :hash_id + remove(:hash_id) end end end diff --git a/apps/db/priv/repo/migrations/20181209205427_videos_providers_as_columns.exs b/apps/db/priv/repo/migrations/20181209205427_videos_providers_as_columns.exs new file mode 100644 index 00000000..d620b8dc --- /dev/null +++ b/apps/db/priv/repo/migrations/20181209205427_videos_providers_as_columns.exs @@ -0,0 +1,62 @@ +defmodule DB.Repo.Migrations.VideosProvidersAsColumns do + @moduledoc """ + This migration changes the `Videos` schema to go from a pair + of {provider, provider_id} to a model where we have multiple `{provider}_id` + column. This will allow to store multiple sources for a single video, with + different offsets to ensure they're all in sync. + """ + + use Ecto.Migration + + def up do + # Add new columns + alter table(:videos) do + add(:youtube_id, :string, null: true, length: 11) + add(:youtube_offset, :integer, null: false, default: 0) + end + + flush() + + # Migrate existing videos - we only have YouTube right now + Ecto.Adapters.SQL.query!(DB.Repo, """ + UPDATE videos SET youtube_id = provider_id; + """) + + flush() + + # Create index + create(unique_index(:videos, :youtube_id)) + + # Remove columns + alter table(:videos) do + remove(:provider) + remove(:provider_id) + end + end + + def down do + # Restore old scheme + alter table(:videos) do + add(:provider, :string) + add(:provider_id, :string) + end + + flush() + + # Migrate existing videos + Ecto.Adapters.SQL.query!(DB.Repo, """ + UPDATE videos SET provider_id = youtube_id, provider = 'youtube'; + """) + + flush() + + # Re-create index + create(unique_index(:videos, [:provider, :provider_id])) + + # Remove columns + alter table(:videos) do + remove(:youtube_id) + remove(:youtube_offset) + end + end +end diff --git a/apps/db/test/support/factory.ex b/apps/db/test/support/factory.ex index 0b56875f..0c19a135 100644 --- a/apps/db/test/support/factory.ex +++ b/apps/db/test/support/factory.ex @@ -50,8 +50,7 @@ defmodule DB.Factory do %Video{ url: "https://www.youtube.com/watch?v=#{youtube_id}", title: Faker.Lorem.sentence(3..8), - provider: "youtube", - provider_id: youtube_id, + youtube_id: youtube_id, hash_id: nil, language: Enum.random(["en", "fr", nil]) } diff --git a/dev/_common.sh b/dev/_common.sh deleted file mode 100644 index 1c9b852b..00000000 --- a/dev/_common.sh +++ /dev/null @@ -1,2 +0,0 @@ -CF_ELIXIR_IMAGE=bitwalker/alpine-elixir-phoenix:1.6.6 -CF_DB_DEV_IMAGE=cf_dev_db diff --git a/dev/check_gitlab_ci.sh b/dev/check_gitlab_ci.sh deleted file mode 100755 index 93c2cd30..00000000 --- a/dev/check_gitlab_ci.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -cd -- "$(dirname $0)/.." - - -echo "Converting YAML file to JSON" -echo "============================" - -CONTENT=$( - cat .gitlab-ci.yml \ - | yaml2json \ - | tr -d '\n' \ - | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' -) - -echo $CONTENT - -echo "" -echo "" -echo "Checking CI config against Gitlab's API" -echo "=======================================" - -curl --header "Content-Type: application/json" https://gitlab.com/api/v4/ci/lint --data "{\"content\": $CONTENT}" \ No newline at end of file diff --git a/dev/check_release.sh b/dev/check_release.sh deleted file mode 100755 index b6314cc5..00000000 --- a/dev/check_release.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -#--------------------------------------------------------------------------------------------------- -# Start a docker release with dev params -# Usage ./test_docker_release.sh -# -# /!\ Database must be started -#--------------------------------------------------------------------------------------------------- - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 app_name" - echo "Example: $0 cf_jobs" - exit 1 -fi - - -TMP_IMAGE_NAME=cf_dev_check_release - -# If any command fails, exit -set -e - -# Build -cd -- "$(dirname $0)" -docker build -t $TMP_IMAGE_NAME --build-arg APP=$1 ../ - -# Run server -echo "Let's test this app! =>" -docker run -it \ - -e "CF_HOST=localhost" \ - -e "CF_SECRET_KEY_BASE=8C6FsJwjV11d+1WPUIbkEH6gB/VavJrcXWoPLujgpclfxjkLkoNFSjVU9XfeNm6s" \ - -e "CF_S3_ACCESS_KEY_ID=test" \ - -e "CF_S3_SECRET_ACCESS_KEY=test" \ - -e "CF_S3_BUCKET=test" \ - -e "CF_DB_HOSTNAME=localhost" \ - -e "CF_DB_USERNAME=postgres" \ - -e "CF_DB_PASSWORD=postgres" \ - -e "CF_DB_NAME=captain_fact_dev" \ - -e "CF_FACEBOOK_APP_ID=506726596325615" \ - -e "CF_FACEBOOK_APP_SECRET=4b320056746b8e57144c889f3baf0424" \ - -e "CF_FRONTEND_URL=http://localhost:3333" \ - -e "CF_CHROME_EXTENSION_ID=chrome-extension://lpdmcoikcclagelhlmibniibjilfifac" \ - -e "CF_BASIC_AUTH_PASSWORD=password" \ - -v "$(pwd)/../priv/secrets:/run/secrets:ro" \ - --network host \ - --rm ${TMP_IMAGE_NAME} console diff --git a/dev/code_style.sh b/dev/code_style.sh deleted file mode 100755 index b5dc2871..00000000 --- a/dev/code_style.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -cd -- "$(dirname $0)/.." -source "./dev/_common.sh" - -docker run -it \ - --rm \ - --workdir=/app \ - --link $CF_DB_DEV_IMAGE:localhost \ - -v `pwd`:/app \ - ${CF_ELIXIR_IMAGE} \ - iex -S mix credo $@ diff --git a/dev/db_gen_migration.sh b/dev/db_gen_migration.sh deleted file mode 100755 index 1fde2800..00000000 --- a/dev/db_gen_migration.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 MigrationName [MigrationParams...]" - echo "----------------------Examples--------------------------" - echo "$0 AddLocaleToInvitationRequest" - exit 1 -fi - -IMAGE=bitwalker/alpine-elixir-phoenix:1.6.0 -cd -- "$(dirname $0)/.." -docker run -it --rm --workdir=/app/apps/db --network=host -v `pwd`:/app ${IMAGE} mix ecto.gen.migration $@ && - echo "Migration created in apps/db/priv/repo/migrations" \ No newline at end of file diff --git a/dev/db_migrate.sh b/dev/db_migrate.sh deleted file mode 100755 index 7074f547..00000000 --- a/dev/db_migrate.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -cd -- "$(dirname $0)/.." -source "./dev/_common.sh" - -docker run -it \ - --rm \ - --workdir=/app \ - --link $CF_DB_DEV_IMAGE:localhost \ - -v `pwd`:/app \ - ${CF_ELIXIR_IMAGE} \ - mix ecto.migrate diff --git a/dev/docker_recreate_db.sh b/dev/docker_recreate_db.sh deleted file mode 100755 index 24162b47..00000000 --- a/dev/docker_recreate_db.sh +++ /dev/null @@ -1,3 +0,0 @@ -docker stop cf_dev_db -docker rm cf_dev_db -docker run -d --name cf_dev_db -p 5432:5432 captainfact/dev-db:latest diff --git a/dev/get_dependencies.sh b/dev/get_dependencies.sh deleted file mode 100755 index 145a6343..00000000 --- a/dev/get_dependencies.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -cd -- "$(dirname $0)/.." -source "./dev/_common.sh" - -docker run -it \ - --rm \ - --workdir=/app \ - -v `pwd`:/app \ - ${CF_ELIXIR_IMAGE} \ - mix deps.get diff --git a/dev/run_command.sh b/dev/run_command.sh deleted file mode 100755 index e9b853c6..00000000 --- a/dev/run_command.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -cd -- "$(dirname $0)/.." -source "./dev/_common.sh" - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 command" - echo "----------------------Examples--------------------------" - echo "Run test suite: $0 mix test" - echo "Start dev server: $0 iex -S mix phx.server" - echo "DB - Run migrations: $0 mix ecto.migrate" - exit 1 -fi - -docker run -it \ - --rm \ - --workdir=/app \ - -p 4000:4000 \ - -p 4001:4001 \ - -p 4002:4002 \ - -p 4003:4003 \ - -p 4004:4004 \ - --link $CF_DB_DEV_IMAGE:localhost \ - -v `pwd`:/app \ - ${CF_ELIXIR_IMAGE} \ - $@ - \ No newline at end of file diff --git a/dev/start_server.sh b/dev/start_server.sh deleted file mode 100755 index a3088428..00000000 --- a/dev/start_server.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -cd -- "$(dirname $0)/.." -source "./dev/_common.sh" - -docker run -it \ - --rm \ - --workdir=/app \ - -p 4000:4000 \ - -p 4001:4001 \ - -p 4002:4002 \ - -p 4003:4003 \ - -p 4004:4004 \ - --link $CF_DB_DEV_IMAGE:localhost \ - -v `pwd`:/app \ - ${CF_ELIXIR_IMAGE} \ - iex -S mix phx.server diff --git a/dev/test.sh b/dev/test.sh deleted file mode 100755 index 1a597607..00000000 --- a/dev/test.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# -# Tests runner. -# -# You can test a specific subset by running ./dev/test.sh test/your_test_subpath -# -# Examples : -# ./dev/test.sh test/db_schema -# ./dev/test.sh test/captain_fact_jobs -# ./dev/test.sh test/captain_fact_jobs/votes_test.exs -# -# ------------------------------------------------------------------------------ - -cd -- "$(dirname $0)/.." -source "./dev/_common.sh" - -docker run -it \ - --rm \ - --workdir=/app \ - --link $CF_DB_DEV_IMAGE:localhost \ - -v `pwd`:/app \ - ${CF_ELIXIR_IMAGE} \ - mix test $@ diff --git a/dev/watch_test.sh b/dev/watch_test.sh deleted file mode 100755 index 3a920df6..00000000 --- a/dev/watch_test.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# -# Tests runner. -# -# You can test a specific subset by running ./dev/test.sh test/your_test_subpath -# -# Examples : -# ./dev/test.sh test/db_schema -# ./dev/test.sh test/captain_fact_jobs -# ./dev/test.sh test/captain_fact_jobs/votes_test.exs -# -# ------------------------------------------------------------------------------ - -cd -- "$(dirname $0)/.." -source "./dev/_common.sh" - -docker run -it \ - --rm \ - --workdir=/app \ - --link $CF_DB_DEV_IMAGE:localhost \ - -v `pwd`:/app \ - ${CF_ELIXIR_IMAGE} \ - mix test.watch $@ diff --git a/rel/release.sh b/rel/release.sh new file mode 100755 index 00000000..095bb963 --- /dev/null +++ b/rel/release.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Build all releases for given tags and push them to Docker registry +# ------------------------------------------------------------------ + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 environment" + echo "Example: $0 staging" + exit 1 +fi + +if [ -z "$DOCKER_USERNAME" ] || [ -z "$DOCKER_PASSWORD" ]; then + echo "[Warning] Docker credentials not provided. You must be logged in to push to registry." +fi + +set -e + +# ---- Images names ---- +CF_API_REST_IMAGE=captainfact/rest-api:$1 +CF_API_GRAPHQL_IMAGE=captainfact/graphql-api:$1 +CF_API_ATOM_FEED=captainfact/atom-feed:$1 +CF_API_OPENGRAPH_IMAGE=captainfact/opengraph:$1 +CF_API_JOBS_IMAGE=captainfact/jobs:$1 + +# ---- Build ---- +echo "[RELEASE] Building Apps 🔨" +docker build --build-arg APP=cf_rest_api -t ${CF_API_REST_IMAGE} . +docker build --build-arg APP=cf_graphql -t ${CF_API_GRAPHQL_IMAGE} . +docker build --build-arg APP=cf_atom_feed -t ${CF_API_ATOM_FEED} . +docker build --build-arg APP=cf_opengraph -t ${CF_API_OPENGRAPH_IMAGE} . +docker build --build-arg APP=cf_jobs -t ${CF_API_JOBS_IMAGE} . + +# ---- Push release ---- +echo "[RELEASE] Pushing Apps 🚀" +docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"; +docker push $CF_API_REST_IMAGE +docker push $CF_API_GRAPHQL_IMAGE +docker push $CF_API_ATOM_FEED +docker push $CF_API_OPENGRAPH_IMAGE +docker push $CF_API_JOBS_IMAGE