diff --git a/.gitignore b/.gitignore
index 5e297107..2b145fc6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,7 @@ until_fail.sh
# Secrets
apps/*/priv/secrets/*
!apps/*/priv/secrets/.keep
+**/*.secret.exs
# Elixir LS
.elixir_ls
diff --git a/.travis.yml b/.travis.yml
index 5c46c86c..83858f57 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,16 +1,7 @@
-# ----- Config -----
-
stages:
- name: test
- name: release
-env:
- CF_API_REST_IMAGE: captainfact/rest-api:$TRAVIS_BRANCH
- CF_API_GRAPHQL_IMAGE: captainfact/graphql-api:$TRAVIS_BRANCH
- CF_API_ATOM_FEED: captainfact/atom-feed:$TRAVIS_BRANCH
- CF_API_OPENGRAPH_IMAGE: captainfact/opengraph:$TRAVIS_BRANCH
- CF_API_JOBS_IMAGE: captainfact/jobs:$TRAVIS_BRANCH
-
# ---- Jobs ----
jobs:
@@ -24,6 +15,7 @@ jobs:
env:
- MIX_ENV=test
before_script:
+ - mix format --check-formatted
- mix local.hex --force
- mix local.rebar --force
- mix deps.get
@@ -31,7 +23,6 @@ jobs:
- mix ecto.migrate
script:
- mix coveralls.travis --umbrella
- - mix format --check-formatted
- stage: release
if: branch IN (master, staging) AND type != pull_request
@@ -39,17 +30,4 @@ jobs:
sudo: required
services: [docker]
script:
- - set -e
- - # ---- Build ----
- - docker build --build-arg APP=cf_rest_api -t ${CF_API_REST_IMAGE} .
- - docker build --build-arg APP=cf_graphql -t ${CF_API_GRAPHQL_IMAGE} .
- - docker build --build-arg APP=cf_atom_feed -t ${CF_API_ATOM_FEED} .
- - docker build --build-arg APP=cf_opengraph -t ${CF_API_OPENGRAPH_IMAGE} .
- - docker build --build-arg APP=cf_jobs -t ${CF_API_JOBS_IMAGE} .
- - # ---- Push release ----
- - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
- - docker push $CF_API_REST_IMAGE
- - docker push $CF_API_GRAPHQL_IMAGE
- - docker push $CF_API_ATOM_FEED
- - docker push $CF_API_OPENGRAPH_IMAGE
- - docker push $CF_API_JOBS_IMAGE
+ - ./rel/release.sh $TRAVIS_BRANCH
diff --git a/README.md b/README.md
index ea8e62cd..3ddef81d 100644
--- a/README.md
+++ b/README.md
@@ -16,27 +16,26 @@
-## Install & Run
+# Install & Run
-### Start DB
+## Prerequisites
+
+You need to install Elixir. We recommand using [asdf](https://github.com/asdf-vm/asdf#setup).
+Check their documentation on how to install it, then run `asdf install` from
+root `captain-fact-api` folder.
+
+## Start DB
Create / launch a postgres instance on your local machine. If you have
docker installed, you can use the pre-seed postgres docker image:
`docker run -d --name cf_dev_db -p 5432:5432 captainfact/dev-db:latest`
-### Start API services
-
-- Without Docker (recommended if you want to make changes in the API)
+## Start API
- - `mix deps.get`
- - `mix ecto.migrate`
- - `iex -S mix`
-
-- With Docker
- - Download project's dependencies with `./dev/get_dependencies.sh`
- - Migrate your database with `./dev/db_migrate.sh`
- - Start server with `./dev/start_server.sh`
+- `mix deps.get` --> Get dependencies
+- `mix ecto.migrate` --> Migrate DB
+- `iex -S mix` --> Start project
Following services will be started:
@@ -46,16 +45,16 @@ Following services will be started:
- [localhost:4003](https://localhost:4003) - GraphQL API (https)
- [localhost:4004](http://localhost:4004) - Atom feed
-You can run tests with `./dev/test.sh`. You can filter which tests to run by
-running something like `./dev/test.sh test/your_test_subpath`.
-Check `./dev/test.sh` script comments for details.
+## Other useful commands
-## Project architecture
+- `mix test` --> Run tests
+- `mix test.watch` --> Run tests watcher
+- `mix format` --> Format code
+- `mix ecto.gen.migration [migration_name]` --> Generate migration
-Elixir offers very nice ways to separate concerns and work with microservices.
-This application is organized as an [umbrella project](https://elixir-lang.org/getting-started/mix-otp/dependencies-and-umbrella-apps.html) which allows us to divide CaptainFact API into small apps.
+# Project architecture
-### File structure
+This application is organized as an [umbrella project](https://elixir-lang.org/getting-started/mix-otp/dependencies-and-umbrella-apps.html) which allows us to divide CaptainFact API into small apps.
```
.
@@ -82,21 +81,9 @@ This application is organized as an [umbrella project](https://elixir-lang.org/g
│ └── config.exs => Releases configuration.
```
-## Styling
-
-Code should follow [Elixy Style Guide](https://github.com/christopheradams/elixir_style_guide)
-and [Credo style guide](https://github.com/rrrene/elixir-style-guide)
-as much as possible.
-
-Avoid lines longer than 80 characters, **never** go beyond 110 characters.
-
-## Linked projects
+# Linked projects
- [Community discussions and documentation](https://github.com/CaptainFact/captain-fact/)
- [Frontend](https://github.com/CaptainFact/captain-fact-frontend)
- [Extension](https://github.com/CaptainFact/captain-fact-extension)
- [Overlay injector](https://github.com/CaptainFact/captain-fact-overlay-injector)
-
-# Feature requests
-
-[![Feature Requests](http://feathub.com/CaptainFact/captain-fact?format=svg)](http://feathub.com/CaptainFact/captain-fact)
diff --git a/apps/cf/config/dev.exs b/apps/cf/config/dev.exs
index 4ddf87ab..0f82cadb 100644
--- a/apps/cf/config/dev.exs
+++ b/apps/cf/config/dev.exs
@@ -31,3 +31,6 @@ config :rollbax,
# Mails
config :cf, CF.Mailer, adapter: Bamboo.LocalAdapter
+
+# Import local secrets if any - use wildcard to ignore errors
+import_config "*dev.secret.exs"
diff --git a/apps/cf/config/test.exs b/apps/cf/config/test.exs
index 400276f9..dfd0f82f 100644
--- a/apps/cf/config/test.exs
+++ b/apps/cf/config/test.exs
@@ -23,9 +23,9 @@ config :cf, CF.Mailer, adapter: Bamboo.TestAdapter
# Reduce the number of round for encryption during tests
config :bcrypt_elixir, :log_rounds, 4
-# Captions mock for testing
-config :cf,
- captions_fetcher: CF.Videos.CaptionsFetcherTest
+# Behaviours mock for testing
+config :cf, captions_fetcher: CF.Videos.CaptionsFetcherTest
+config :cf, use_test_video_metadata_fetcher: true
# Configure Rollbar (errors reporting)
config :rollbax,
diff --git a/apps/cf/lib/accounts/accounts.ex b/apps/cf/lib/accounts/accounts.ex
index 8ff4c878..56dee09d 100644
--- a/apps/cf/lib/accounts/accounts.ex
+++ b/apps/cf/lib/accounts/accounts.ex
@@ -48,11 +48,27 @@ defmodule CF.Accounts do
# Do create user
user_params
+ |> prepare_user_params_from_third_party()
|> create_account_from_params(provider_params, allow_empty_username)
|> after_create(invitation)
end
end
+ # Special formating for third-party provided user params
+ defp prepare_user_params_from_third_party(params) do
+ # Truncate name to avoid crashing when registering with a too-long name
+ cond do
+ Map.has_key?(params, :name) ->
+ Map.update(params, :name, nil, &String.slice(&1, 0..19))
+
+ Map.has_key?(params, "name") ->
+ Map.update(params, "name", nil, &String.slice(&1, 0..19))
+
+ true ->
+ params
+ end
+ end
+
defp create_account_from_params(user_params, provider_params, allow_empty_username) do
case Map.get(user_params, "username") || Map.get(user_params, :username) do
username when allow_empty_username and (is_nil(username) or username == "") ->
diff --git a/apps/cf/lib/actions/action_creator.ex b/apps/cf/lib/actions/action_creator.ex
index 182c2be3..0c5ee518 100644
--- a/apps/cf/lib/actions/action_creator.ex
+++ b/apps/cf/lib/actions/action_creator.ex
@@ -100,7 +100,7 @@ defmodule CF.Actions.ActionCreator do
user_id,
:video,
:update,
- video_id: video.video_id,
+ video_id: video.id,
changes: changes
)
end
diff --git a/apps/cf/lib/videos/captions_fetcher.ex b/apps/cf/lib/videos/captions_fetcher.ex
index 7ba7be4f..649d294f 100644
--- a/apps/cf/lib/videos/captions_fetcher.ex
+++ b/apps/cf/lib/videos/captions_fetcher.ex
@@ -3,6 +3,5 @@ defmodule CF.Videos.CaptionsFetcher do
Fetch captions for videos.
"""
- @callback fetch(String.t(), String.t()) ::
- {:ok, DB.Schema.VideoCaption.t()} | {:error, binary()}
+ @callback fetch(DB.Schema.Video.t()) :: {:ok, DB.Schema.VideoCaption.t()} | {:error, binary()}
end
diff --git a/apps/cf/lib/videos/captions_fetcher_test.ex b/apps/cf/lib/videos/captions_fetcher_test.ex
index 9295049a..178e08ca 100644
--- a/apps/cf/lib/videos/captions_fetcher_test.ex
+++ b/apps/cf/lib/videos/captions_fetcher_test.ex
@@ -6,7 +6,7 @@ defmodule CF.Videos.CaptionsFetcherTest do
@behaviour CF.Videos.CaptionsFetcher
@impl true
- def fetch(_provider_id, _locale) do
+ def fetch(_video) do
captions = %DB.Schema.VideoCaption{
content: "__TEST-CONTENT__",
format: "xml"
diff --git a/apps/cf/lib/videos/captions_fetcher_youtube.ex b/apps/cf/lib/videos/captions_fetcher_youtube.ex
index 4403924d..5081c689 100644
--- a/apps/cf/lib/videos/captions_fetcher_youtube.ex
+++ b/apps/cf/lib/videos/captions_fetcher_youtube.ex
@@ -6,7 +6,7 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
@behaviour CF.Videos.CaptionsFetcher
@impl true
- def fetch(youtube_id, locale) do
+ def fetch(%{youtube_id: youtube_id, locale: locale}) do
with {:ok, content} <- fetch_captions_content(youtube_id, locale) do
captions = %DB.Schema.VideoCaption{
content: content,
diff --git a/apps/cf/lib/videos/metadata_fetcher.ex b/apps/cf/lib/videos/metadata_fetcher.ex
index 96dda2e9..9df30716 100644
--- a/apps/cf/lib/videos/metadata_fetcher.ex
+++ b/apps/cf/lib/videos/metadata_fetcher.ex
@@ -1,76 +1,16 @@
defmodule CF.Videos.MetadataFetcher do
@moduledoc """
- Methods to fetch metadata (title, language) from videos
+ Fetch metadata for video.
"""
- require Logger
-
- alias Kaur.Result
- alias GoogleApi.YouTube.V3.Connection, as: YouTubeConnection
- alias GoogleApi.YouTube.V3.Api.Videos, as: YouTubeVideos
- alias GoogleApi.YouTube.V3.Model.Video, as: YouTubeVideo
- alias GoogleApi.YouTube.V3.Model.VideoListResponse, as: YouTubeVideoList
-
- alias DB.Schema.Video
+ @type video_metadata :: %{
+ title: String.t(),
+ language: String.t(),
+ url: String.t()
+ }
@doc """
- Fetch metadata from video. Returns an object containing :title, :url and :language
-
- Usage:
- iex> fetch_video_metadata("https://www.youtube.com/watch?v=OhWRT3PhMJs")
- iex> fetch_video_metadata({"youtube", "OhWRT3PhMJs"})
+ Takes an URL, fetch the metadata and return them
"""
- def fetch_video_metadata(nil),
- do: {:error, "Invalid URL"}
-
- if Application.get_env(:db, :env) == :test do
- def fetch_video_metadata(url = "__TEST__/" <> _id) do
- {:ok, %{title: "__TEST-TITLE__", url: url}}
- end
- end
-
- def fetch_video_metadata(url) when is_binary(url),
- do: fetch_video_metadata(Video.parse_url(url))
-
- def fetch_video_metadata({"youtube", provider_id}) do
- case Application.get_env(:cf, :youtube_api_key) do
- nil ->
- Logger.warn("No YouTube API key provided. Falling back to HTML fetcher")
- fetch_video_metadata_html("youtube", provider_id)
-
- api_key ->
- fetch_video_metadata_api("youtube", provider_id, api_key)
- end
- end
-
- defp fetch_video_metadata_api("youtube", provider_id, api_key) do
- YouTubeConnection.new()
- |> YouTubeVideos.youtube_videos_list("snippet", id: provider_id, key: api_key)
- |> Result.map_error(fn e -> "YouTube API Error: #{inspect(e)}" end)
- |> Result.keep_if(&(!Enum.empty?(&1.items)), "Video doesn't exist")
- |> Result.map(fn %YouTubeVideoList{items: [video = %YouTubeVideo{} | _]} ->
- %{
- title: video.snippet.title,
- language: video.snippet.defaultLanguage || video.snippet.defaultAudioLanguage,
- url: Video.build_url(%{provider: "youtube", provider_id: provider_id})
- }
- end)
- end
-
- defp fetch_video_metadata_html(provider, id) do
- url = Video.build_url(%{provider: provider, provider_id: id})
-
- case HTTPoison.get(url) do
- {:ok, %HTTPoison.Response{body: body}} ->
- meta = Floki.attribute(body, "meta[property='og:title']", "content")
-
- case meta do
- [] -> {:error, "Page does not contains an OpenGraph title attribute"}
- [title] -> {:ok, %{title: HtmlEntities.decode(title), url: url}}
- end
-
- {_, _} ->
- {:error, "Remote URL didn't respond correctly"}
- end
- end
+ @callback fetch_video_metadata(String.t()) :: {:ok, video_metadata} | {:error, binary()}
end
diff --git a/apps/cf/lib/videos/metadata_fetcher_opengraph.ex b/apps/cf/lib/videos/metadata_fetcher_opengraph.ex
new file mode 100644
index 00000000..61de6ecd
--- /dev/null
+++ b/apps/cf/lib/videos/metadata_fetcher_opengraph.ex
@@ -0,0 +1,25 @@
+defmodule CF.Videos.MetadataFetcher.Opengraph do
+ @moduledoc """
+ Methods to fetch metadata (title, language) from videos
+ """
+
+ @behaviour CF.Videos.MetadataFetcher
+
+ @doc """
+ Fetch metadata from video using OpenGraph tags.
+ """
+ def fetch_video_metadata(url) do
+ case HTTPoison.get(url) do
+ {:ok, %HTTPoison.Response{body: body}} ->
+ meta = Floki.attribute(body, "meta[property='og:title']", "content")
+
+ case meta do
+ [] -> {:error, "Page does not contains an OpenGraph title attribute"}
+ [title] -> {:ok, %{title: HtmlEntities.decode(title), url: url}}
+ end
+
+ {_, _} ->
+ {:error, "Remote URL didn't respond correctly"}
+ end
+ end
+end
diff --git a/apps/cf/lib/videos/metadata_fetcher_test.ex b/apps/cf/lib/videos/metadata_fetcher_test.ex
new file mode 100644
index 00000000..948767cf
--- /dev/null
+++ b/apps/cf/lib/videos/metadata_fetcher_test.ex
@@ -0,0 +1,18 @@
+defmodule CF.Videos.MetadataFetcher.Test do
+ @moduledoc """
+ Methods to fetch metadata (title, language) from videos
+ """
+
+ @behaviour CF.Videos.MetadataFetcher
+
+ @doc """
+ Fetch metadata from video using OpenGraph tags.
+ """
+ def fetch_video_metadata(url) do
+ {:ok,
+ %{
+ title: "__TEST-TITLE__",
+ url: url
+ }}
+ end
+end
diff --git a/apps/cf/lib/videos/metadata_fetcher_youtube.ex b/apps/cf/lib/videos/metadata_fetcher_youtube.ex
new file mode 100644
index 00000000..0a20ae01
--- /dev/null
+++ b/apps/cf/lib/videos/metadata_fetcher_youtube.ex
@@ -0,0 +1,51 @@
+defmodule CF.Videos.MetadataFetcher.Youtube do
+ @moduledoc """
+ Methods to fetch metadata (title, language) from videos
+ """
+
+ @behaviour CF.Videos.MetadataFetcher
+
+ require Logger
+
+ alias Kaur.Result
+ alias GoogleApi.YouTube.V3.Connection, as: YouTubeConnection
+ alias GoogleApi.YouTube.V3.Api.Videos, as: YouTubeVideos
+ alias GoogleApi.YouTube.V3.Model.Video, as: YouTubeVideo
+ alias GoogleApi.YouTube.V3.Model.VideoListResponse, as: YouTubeVideoList
+
+ alias DB.Schema.Video
+ alias CF.Videos.MetadataFetcher
+
+ @doc """
+ Fetch metadata from video. Returns an object containing :title, :url and :language
+ """
+ def fetch_video_metadata(nil),
+ do: {:error, "Invalid URL"}
+
+ def fetch_video_metadata(url) when is_binary(url) do
+ {:youtube, youtube_id} = Video.parse_url(url)
+
+ case Application.get_env(:cf, :youtube_api_key) do
+ nil ->
+ Logger.warn("No YouTube API key provided. Falling back to HTML fetcher")
+ MetadataFetcher.Opengraph.fetch_video_metadata(url)
+
+ api_key ->
+ do_fetch_video_metadata(youtube_id, api_key)
+ end
+ end
+
+ defp do_fetch_video_metadata(youtube_id, api_key) do
+ YouTubeConnection.new()
+ |> YouTubeVideos.youtube_videos_list("snippet", id: youtube_id, key: api_key)
+ |> Result.map_error(fn e -> "YouTube API Error: #{inspect(e)}" end)
+ |> Result.keep_if(&(!Enum.empty?(&1.items)), "remote_video_404")
+ |> Result.map(fn %YouTubeVideoList{items: [video = %YouTubeVideo{} | _]} ->
+ %{
+ title: video.snippet.title,
+ language: video.snippet.defaultLanguage || video.snippet.defaultAudioLanguage,
+ url: Video.build_url(%{youtube_id: youtube_id})
+ }
+ end)
+ end
+end
diff --git a/apps/cf/lib/videos/videos.ex b/apps/cf/lib/videos/videos.ex
index 1facbbf2..e3129af1 100644
--- a/apps/cf/lib/videos/videos.ex
+++ b/apps/cf/lib/videos/videos.ex
@@ -6,17 +6,18 @@ defmodule CF.Videos do
import Ecto.Query, warn: false
import CF.Videos.MetadataFetcher
import CF.Videos.CaptionsFetcher
+ import CF.Actions.ActionCreator, only: [action_update: 2]
alias Ecto.Multi
alias DB.Repo
alias DB.Schema.Video
- alias DB.Schema.Statement
alias DB.Schema.Speaker
alias DB.Schema.VideoSpeaker
alias DB.Schema.VideoCaption
alias CF.Actions.ActionCreator
alias CF.Accounts.UserPermissions
+ alias CF.Videos.MetadataFetcher
@captions_fetcher Application.get_env(:cf, :captions_fetcher)
@@ -34,28 +35,17 @@ defmodule CF.Videos do
def videos_list(filters, false),
do: Repo.all(Video.query_list(Video, filters))
- @doc """
- Index videos, returning only their id, provider_id and provider.
- Accepted filters are the same than for `videos_list/1`
- """
- def videos_index(from_id \\ 0) do
- Video
- |> select([v], %{id: v.id, provider: v.provider, provider_id: v.provider_id})
- |> where([v], v.id > ^from_id)
- |> Repo.all()
- end
-
@doc """
Return the corresponding video if it has already been added, `nil` otherwise
"""
def get_video_by_url(url) do
case Video.parse_url(url) do
- {provider, id} ->
+ {:youtube, id} ->
Video
|> Video.with_speakers()
- |> Repo.get_by(provider: provider, provider_id: id)
+ |> Repo.get_by(youtube_id: id)
- nil ->
+ _ ->
nil
end
end
@@ -74,7 +64,8 @@ defmodule CF.Videos do
def create!(user, video_url, is_partner \\ nil) do
UserPermissions.check!(user, :add, :video)
- with {:ok, metadata} <- fetch_video_metadata(video_url) do
+ with metadata_fetcher when not is_nil(metadata_fetcher) <- get_metadata_fetcher(video_url),
+ {:ok, metadata} <- metadata_fetcher.(video_url) do
# Videos posted by publishers are recorded as partner unless explicitely
# specified otherwise (false)
base_video = %Video{is_partner: user.is_publisher && is_partner != false}
@@ -106,33 +97,20 @@ defmodule CF.Videos do
Returns {:ok, statements} if success, {:error, reason} otherwise.
Returned statements contains only an id and a key
"""
- def shift_statements(user, video_id, offset) when is_integer(offset) do
+ def shift_statements(user, video_id, offsets) do
UserPermissions.check!(user, :update, :video)
- statements_query = where(Statement, [s], s.video_id == ^video_id)
+ video = Repo.get!(Video, video_id)
+ changeset = Video.changeset_shift_offsets(video, offsets)
Multi.new()
- |> Multi.update_all(
- :statements_update,
- statements_query,
- [inc: [time: offset]],
- returning: [:id, :time]
- )
- |> Multi.insert(
- :action,
- ActionCreator.action(
- user.id,
- :video,
- :update,
- video_id: video_id,
- changes: %{"statements_time" => offset}
- )
- )
+ |> Multi.update(:video, changeset)
+ |> Multi.insert(:action_update, action_update(user.id, changeset))
|> Repo.transaction()
|> case do
- {:ok, %{statements_update: {_, statements}}} ->
- {:ok, Enum.map(statements, &%{id: &1.id, time: &1.time})}
+ {:ok, %{video: video}} ->
+ {:ok, video}
- {:error, _, reason, _} ->
+ {:error, _operation, reason, _changes} ->
{:error, reason}
end
end
@@ -163,10 +141,9 @@ defmodule CF.Videos do
Usage:
iex> download_captions(video)
- iex> download_captions(video)
"""
def download_captions(video = %Video{}) do
- with {:ok, captions} <- @captions_fetcher.fetch(video.provider_id, video.language) do
+ with {:ok, captions} <- @captions_fetcher.fetch(video) do
captions
|> VideoCaption.changeset(%{video_id: video.id})
|> Repo.insert()
@@ -174,4 +151,20 @@ defmodule CF.Videos do
{:ok, captions}
end
end
+
+ defp get_metadata_fetcher(video_url) do
+ cond do
+ Application.get_env(:cf, :use_test_video_metadata_fetcher) ->
+ &MetadataFetcher.Test.fetch_video_metadata/1
+
+ # We only support YouTube for now
+ # TODO Use a Regex here
+ video_url ->
+ &MetadataFetcher.Youtube.fetch_video_metadata/1
+
+ # Use a default fetcher that retrieves info from OpenGraph tags
+ true ->
+ &MetadataFetcher.Opengraph.fetch_video_metadata/1
+ end
+ end
end
diff --git a/apps/cf/mix.exs b/apps/cf/mix.exs
index 221f525c..b9ef4810 100644
--- a/apps/cf/mix.exs
+++ b/apps/cf/mix.exs
@@ -4,7 +4,7 @@ defmodule CF.Mixfile do
def project do
[
app: :cf,
- version: "0.9.0",
+ version: "0.9.1",
build_path: "../../_build",
compilers: [:phoenix, :gettext] ++ Mix.compilers(),
config_path: "../../config/config.exs",
diff --git a/apps/cf/priv/secrets/.keep b/apps/cf/priv/secrets/.keep
deleted file mode 100644
index e69de29b..00000000
diff --git a/apps/cf/test/accounts/accounts_test.exs b/apps/cf/test/accounts/accounts_test.exs
index ecf35f5f..a333dc88 100644
--- a/apps/cf/test/accounts/accounts_test.exs
+++ b/apps/cf/test/accounts/accounts_test.exs
@@ -162,6 +162,13 @@ defmodule CF.AccountsTest do
Accounts.create_account(user_params, invit.token, provider_params: provider_params)
end
+ test "truncate name if too long" do
+ user_params = Map.put(build_user_params(), :name, "abcdefghijklmnopqrstuvwxyz")
+ provider_params = %{fb_user_id: "4242424242"}
+ {:ok, user} = Accounts.create_account(user_params, nil, provider_params: provider_params)
+ assert user.name == "abcdefghijklmnopqrst"
+ end
+
test "delete invitation request after creating the user" do
Repo.delete_all(DB.Schema.InvitationRequest)
invit = insert(:invitation_request)
diff --git a/apps/cf/test/videos/videos_test.exs b/apps/cf/test/videos/videos_test.exs
index 6efde826..e70cefa9 100644
--- a/apps/cf/test/videos/videos_test.exs
+++ b/apps/cf/test/videos/videos_test.exs
@@ -5,7 +5,7 @@ defmodule CF.VideosTest do
alias CF.Videos
alias CF.Accounts.UserPermissions.PermissionsError
- defp test_url, do: "__TEST__/#{DB.Utils.TokenGenerator.generate(8)}"
+ defp test_url, do: "https://www.youtube.com/watch?v=#{DB.Utils.TokenGenerator.generate(11)}"
describe "Add video" do
test "without enough reputation" do
@@ -59,7 +59,13 @@ defmodule CF.VideosTest do
describe "Fetch captions" do
test "fetch captions" do
- video = DB.Factory.insert(:video, provider: "__TEST__", language: "en")
+ video =
+ DB.Factory.insert(
+ :video,
+ youtube_id: DB.Utils.TokenGenerator.generate(11),
+ language: "en"
+ )
+
{:ok, captions} = Videos.download_captions(video)
assert captions.content == "__TEST-CONTENT__"
diff --git a/apps/cf_atom_feed/lib/flags.ex b/apps/cf_atom_feed/lib/flags.ex
new file mode 100644
index 00000000..deb2d05b
--- /dev/null
+++ b/apps/cf_atom_feed/lib/flags.ex
@@ -0,0 +1,90 @@
+defmodule CF.AtomFeed.Flags do
+ @moduledoc """
+ Generate an ATOM feed that contains all flags
+ """
+
+ import Ecto.Query
+
+ alias Atomex.{Feed, Entry}
+ alias DB.Schema.{Flag, UserAction, User, Comment}
+ alias DB.Type.FlagReason
+ alias CF.Utils.FrontendRouter
+
+ @nb_items_max 50
+
+ @doc """
+ Get an ATOM feed containing all site's flags in reverse chronological
+ order (newest first)
+ """
+ def feed_all() do
+ flags = fetch_flags()
+ generate_feed(flags, last_update(flags))
+ end
+
+ defp fetch_flags() do
+ DB.Repo.all(
+ from(
+ flag in Flag,
+ order_by: [desc: flag.inserted_at],
+ left_join: action in assoc(flag, :action),
+ left_join: comment in assoc(action, :comment),
+ left_join: user in assoc(comment, :user),
+ left_join: source in assoc(comment, :source),
+ preload: [action: [comment: [:user, :statement, :source]]],
+ limit: @nb_items_max
+ )
+ )
+ end
+
+ defp last_update(_flags = [flag | _]),
+ do: DateTime.from_naive!(flag.inserted_at, "Etc/UTC")
+
+ defp last_update(_),
+ do: DateTime.utc_now()
+
+ defp generate_feed(flags, last_update) do
+ FrontendRouter.base_url()
+ |> Feed.new(last_update, "[CaptainFact] All Flags")
+ |> CF.AtomFeed.Common.feed_author()
+ |> Feed.link("https://feed.captainfact.io/flags/", rel: "self")
+ |> Feed.entries(Enum.map(flags, &get_entry/1))
+ |> Feed.build()
+ |> Atomex.generate_document()
+ end
+
+ defp get_entry(flag) do
+ comment = flag.action.comment
+ user_appelation = User.user_appelation(comment.user)
+ title = entry_title(flag, user_appelation)
+ link = comment_url(flag.action)
+ insert_datetime = DateTime.from_naive!(flag.inserted_at, "Etc/UTC")
+
+ link
+ |> Entry.new(insert_datetime, title)
+ |> Entry.link(link)
+ |> Entry.published(insert_datetime)
+ |> Entry.content("""
+ ```
+ #{comment.text}
+ ```
+ Source Comment: #{source(comment)}\n
+ Flag reason: #{FlagReason.label(flag.reason)}
+ """)
+ |> Entry.build()
+ end
+
+ defp source(%{source: nil}),
+ do: "None"
+
+ defp source(%{source: %{url: url, site_name: site_name}}),
+ do: "[#{site_name || url}](#{url})"
+
+ defp entry_title(flag, user_appelation) do
+ "New Flag for #{user_appelation} comment ##{flag.action.comment.id}"
+ end
+
+ defp comment_url(action) do
+ video_hash_id = DB.Type.VideoHashId.encode(action.video_id)
+ FrontendRouter.comment_url(video_hash_id, action.comment)
+ end
+end
diff --git a/apps/cf_atom_feed/lib/router.ex b/apps/cf_atom_feed/lib/router.ex
index 771a931a..40bd8c2b 100644
--- a/apps/cf_atom_feed/lib/router.ex
+++ b/apps/cf_atom_feed/lib/router.ex
@@ -42,4 +42,8 @@ defmodule CF.AtomFeed.Router do
get "/videos" do
render_feed(conn, CF.AtomFeed.Videos.feed_all())
end
+
+ get "/flags" do
+ render_feed(conn, CF.AtomFeed.Flags.feed_all())
+ end
end
diff --git a/apps/cf_atom_feed/lib/videos.ex b/apps/cf_atom_feed/lib/videos.ex
index f45dbb3a..18bdfe22 100644
--- a/apps/cf_atom_feed/lib/videos.ex
+++ b/apps/cf_atom_feed/lib/videos.ex
@@ -40,30 +40,28 @@ defmodule CF.AtomFeed.Videos do
defp render_entry(video) do
video_link = FrontendRouter.video_url(video.hash_id)
- insert_datetime = video_datetime(video)
video_link
- |> Entry.new(insert_datetime, video.title)
+ |> Entry.new(video.inserted_at, video.title)
|> Entry.link(video_link)
- |> Entry.published(insert_datetime)
+ |> Entry.published(video.inserted_at)
|> Entry.content(entry_content(video))
|> Entry.build()
end
+ defp entry_content(%{speakers: []}) do
+ ""
+ end
+
defp entry_content(video) do
- """
- Speakers: #{Enum.map_join(video.speakers, ", ", &render_speaker/1)}
- """
+ Enum.map_join(video.speakers, ", ", &render_speaker/1)
end
defp render_speaker(speaker = %{full_name: name}),
do: "[#{name}](#{FrontendRouter.speaker_url(speaker)})"
- defp video_datetime(%{inserted_at: naive_datetime}),
- do: DateTime.from_naive!(naive_datetime, "Etc/UTC")
-
defp last_update([video | _]),
- do: video_datetime(video)
+ do: video.inserted_at
defp last_update(_),
do: DateTime.utc_now()
diff --git a/apps/cf_atom_feed/mix.exs b/apps/cf_atom_feed/mix.exs
index 9fbeb790..daf6d507 100644
--- a/apps/cf_atom_feed/mix.exs
+++ b/apps/cf_atom_feed/mix.exs
@@ -4,7 +4,7 @@ defmodule CF.AtomFeed.Mixfile do
def project do
[
app: :cf_atom_feed,
- version: "0.9.0",
+ version: "0.9.1",
build_path: "../../_build",
config_path: "../../config/config.exs",
deps_path: "../../deps",
diff --git a/apps/cf_graphql/lib/resolvers/videos.ex b/apps/cf_graphql/lib/resolvers/videos.ex
index a3dd658c..e60bd209 100644
--- a/apps/cf_graphql/lib/resolvers/videos.ex
+++ b/apps/cf_graphql/lib/resolvers/videos.ex
@@ -15,21 +15,21 @@ defmodule CF.GraphQL.Resolvers.Videos do
# Queries
def get(_root, %{id: id}, _info) do
- case get_video_by_id(id) do
+ case CF.Videos.get_video_by_id(id) do
nil -> {:error, "Video #{id} doesn't exist"}
video -> {:ok, video}
end
end
def get(_root, %{hash_id: id}, _info) do
- case get_video_by_id(id) do
+ case CF.Videos.get_video_by_id(id) do
nil -> {:error, "Video #{id} doesn't exist"}
video -> {:ok, video}
end
end
def get(_root, %{url: url}, _info) do
- case get_video_by_url(url) do
+ case CF.Videos.get_video_by_url(url) do
nil -> {:error, "Video with url #{url} doesn't exist"}
video -> {:ok, video}
end
@@ -69,20 +69,4 @@ defmodule CF.GraphQL.Resolvers.Videos do
|> Repo.all()
|> Enum.group_by(& &1.video_id)
end
-
- # ---- Private ----
-
- defp get_video_by_url(url) do
- case Video.parse_url(url) do
- {provider, id} ->
- Video
- |> Video.with_speakers()
- |> Repo.get_by(provider: provider, provider_id: id)
-
- nil ->
- nil
- end
- end
-
- defp get_video_by_id(id), do: Repo.get(Video, id)
end
diff --git a/apps/cf_graphql/lib/schema/types/video.ex b/apps/cf_graphql/lib/schema/types/video.ex
index ff7698a9..e104e889 100644
--- a/apps/cf_graphql/lib/schema/types/video.ex
+++ b/apps/cf_graphql/lib/schema/types/video.ex
@@ -21,9 +21,15 @@ defmodule CF.GraphQL.Schema.Types.Video do
@desc "Video URL"
field(:url, non_null(:string), do: resolve(&Resolvers.Videos.url/3))
@desc "Video provider (youtube, vimeo...etc)"
- field(:provider, non_null(:string))
+ field(:provider, non_null(:string), deprecate: "Use `url` or `youtube_id`") do
+ resolve(fn _, _, _ -> {:ok, "youtube"} end)
+ end
+
@desc "Unique ID used to identify video with provider"
- field(:provider_id, non_null(:string))
+ field(:provider_id, non_null(:string), deprecate: "Use `url` or `youtube_id`") do
+ resolve(fn v, _, _ -> {:ok, v.youtube_id} end)
+ end
+
@desc "Language of the video represented as a two letters locale"
field(:language, :string)
@desc "Video insert datetime"
@@ -41,6 +47,14 @@ defmodule CF.GraphQL.Schema.Types.Video do
resolve(&Resolvers.Videos.statements/3)
complexity(join_complexity())
end
+
+ # Video providers
+
+ @desc "Youtube ID for this video"
+ field(:youtube_id, :string)
+
+ @desc "Offset for all statements on this video when watched with YouTube player"
+ field(:youtube_offset, non_null(:integer))
end
@desc "A list a paginated videos"
diff --git a/apps/cf_graphql/mix.exs b/apps/cf_graphql/mix.exs
index 1257023b..319c9490 100644
--- a/apps/cf_graphql/mix.exs
+++ b/apps/cf_graphql/mix.exs
@@ -4,7 +4,7 @@ defmodule CF.GraphQL.Mixfile do
def project do
[
app: :cf_graphql,
- version: "0.9.0",
+ version: "0.9.1",
build_path: "../../_build",
config_path: "../../config/config.exs",
deps_path: "../../deps",
diff --git a/apps/cf_jobs/config/test.exs b/apps/cf_jobs/config/test.exs
index d33b433c..5305a6d5 100644
--- a/apps/cf_jobs/config/test.exs
+++ b/apps/cf_jobs/config/test.exs
@@ -1,4 +1,4 @@
use Mix.Config
# Disable CRON tasks on test
-config :cf_jobs, CF.Scheduler, jobs: []
+config :cf_jobs, CF.Jobs.Scheduler, jobs: []
diff --git a/apps/cf_jobs/mix.exs b/apps/cf_jobs/mix.exs
index 4e8e2fd9..eba4950b 100644
--- a/apps/cf_jobs/mix.exs
+++ b/apps/cf_jobs/mix.exs
@@ -4,7 +4,7 @@ defmodule CF.Jobs.Mixfile do
def project do
[
app: :cf_jobs,
- version: "0.9.0",
+ version: "0.9.1",
build_path: "../../_build",
config_path: "../../config/config.exs",
deps_path: "../../deps",
diff --git a/apps/cf_opengraph/mix.exs b/apps/cf_opengraph/mix.exs
index 3d2cc9fe..7ae53f2a 100644
--- a/apps/cf_opengraph/mix.exs
+++ b/apps/cf_opengraph/mix.exs
@@ -4,7 +4,7 @@ defmodule CF.Opengraph.MixProject do
def project do
[
app: :cf_opengraph,
- version: "0.9.0",
+ version: "0.9.1",
elixir: "~> 1.6",
start_permanent: Mix.env() == :prod,
deps: deps(),
diff --git a/apps/cf_rest_api/lib/channels/statements_channel.ex b/apps/cf_rest_api/lib/channels/statements_channel.ex
index cb7dd1e7..94b5850a 100644
--- a/apps/cf_rest_api/lib/channels/statements_channel.ex
+++ b/apps/cf_rest_api/lib/channels/statements_channel.ex
@@ -10,7 +10,6 @@ defmodule CF.RestApi.StatementsChannel do
alias DB.Schema.Statement
alias CF.Statements
- alias CF.Videos
alias CF.Accounts.UserPermissions
alias CF.RestApi.{StatementView, ErrorView}
@@ -32,26 +31,6 @@ defmodule CF.RestApi.StatementsChannel do
handle_in_authenticated(command, params, socket, &handle_in_authenticated!/3)
end
- @doc """
- Shift all video's statements
- """
- def handle_in_authenticated!("shift_all", offset, socket) do
- user = Repo.get(DB.Schema.User, socket.assigns.user_id)
-
- case Videos.shift_statements(
- user,
- socket.assigns.video_id,
- String.to_integer(offset)
- ) do
- {:ok, statements} ->
- broadcast!(socket, "statements_updated", %{statements: statements})
- {:reply, :ok, socket}
-
- {:error, _} ->
- {:reply, :error, socket}
- end
- end
-
@doc """
Add a new statement
"""
diff --git a/apps/cf_rest_api/lib/channels/user_socket.ex b/apps/cf_rest_api/lib/channels/user_socket.ex
index 718719f6..27f38e34 100644
--- a/apps/cf_rest_api/lib/channels/user_socket.ex
+++ b/apps/cf_rest_api/lib/channels/user_socket.ex
@@ -16,6 +16,7 @@ defmodule CF.RestApi.UserSocket do
## Transports
transport(:websocket, Phoenix.Transports.WebSocket)
+ transport(:longpoll, Phoenix.Transports.LongPoll)
# Connect with token
def connect(%{"token" => token}, socket) do
diff --git a/apps/cf_rest_api/lib/channels/video_debate_channel.ex b/apps/cf_rest_api/lib/channels/video_debate_channel.ex
index ba47e116..fd63a65c 100644
--- a/apps/cf_rest_api/lib/channels/video_debate_channel.ex
+++ b/apps/cf_rest_api/lib/channels/video_debate_channel.ex
@@ -18,6 +18,7 @@ defmodule CF.RestApi.VideoDebateChannel do
alias DB.Schema.Speaker
alias DB.Schema.VideoSpeaker
+ alias CF.Videos
alias CF.Accounts.UserPermissions
alias CF.RestApi.{VideoView, SpeakerView, ChangesetView}
@@ -58,6 +59,27 @@ defmodule CF.RestApi.VideoDebateChannel do
handle_in_authenticated(command, params, socket, &handle_in_authenticated!/3)
end
+ @doc """
+ Shift all video's statements
+ """
+ def handle_in_authenticated!("shift_statements", offsets, socket) do
+ user = Repo.get(DB.Schema.User, socket.assigns.user_id)
+
+ case Videos.shift_statements(user, socket.assigns.video_id, offsets) do
+ {:ok, video} ->
+ rendered_video =
+ video
+ |> DB.Repo.preload(:speakers)
+ |> View.render_one(VideoView, "video.json")
+
+ broadcast!(socket, "video_updated", %{video: rendered_video})
+ {:reply, :ok, socket}
+
+ {:error, _} ->
+ {:reply, :error, socket}
+ end
+ end
+
@doc """
Add an existing speaker to the video
"""
diff --git a/apps/cf_rest_api/lib/controllers/video_controller.ex b/apps/cf_rest_api/lib/controllers/video_controller.ex
index 690fb94a..21895037 100644
--- a/apps/cf_rest_api/lib/controllers/video_controller.ex
+++ b/apps/cf_rest_api/lib/controllers/video_controller.ex
@@ -5,6 +5,7 @@ defmodule CF.RestApi.VideoController do
use CF.RestApi, :controller
import CF.Videos
+ alias CF.RestApi.ChangesetView
action_fallback(CF.RestApi.FallbackController)
@@ -23,16 +24,6 @@ defmodule CF.RestApi.VideoController do
def index(conn, _params),
do: render(conn, :index, videos: videos_list())
- @doc """
- List all videos but only return indexes.
- """
- @deprecated "Extension/Injector is now using GraphQL API for this"
- def index_ids(conn, %{"min_id" => min_id}),
- do: json(conn, videos_index(min_id))
-
- def index_ids(conn, _),
- do: json(conn, videos_index())
-
@doc """
Create a new video based on `url`.
If it already exist, just returns the video.
@@ -44,10 +35,15 @@ defmodule CF.RestApi.VideoController do
|> Guardian.Plug.current_resource()
|> create!(url, params["is_partner"])
|> case do
- {:error, message} ->
+ {:error, error} when is_binary(error) ->
+ conn
+ |> put_status(:unprocessable_entity)
+ |> json(%{error: error})
+
+ {:error, changeset = %Ecto.Changeset{}} ->
conn
|> put_status(:unprocessable_entity)
- |> json(%{error: %{url: message}})
+ |> json(ChangesetView.render("error.json", %{changeset: changeset}))
{:ok, video} ->
render(conn, "show.json", video: video)
diff --git a/apps/cf_rest_api/lib/router.ex b/apps/cf_rest_api/lib/router.ex
index dda93eee..cba25be3 100644
--- a/apps/cf_rest_api/lib/router.ex
+++ b/apps/cf_rest_api/lib/router.ex
@@ -24,7 +24,6 @@ defmodule CF.RestApi.Router do
# ---- Public endpoints ----
get("/", ApiInfoController, :get)
get("/videos", VideoController, :index)
- get("/videos/index", VideoController, :index_ids)
get("/speakers/:slug_or_id", SpeakerController, :show)
post("/search/video", VideoController, :search)
get("/videos/:video_id/statements", StatementController, :get)
diff --git a/apps/cf_rest_api/lib/views/video_view.ex b/apps/cf_rest_api/lib/views/video_view.ex
index 5d21685f..89bb5c66 100644
--- a/apps/cf_rest_api/lib/views/video_view.ex
+++ b/apps/cf_rest_api/lib/views/video_view.ex
@@ -14,8 +14,10 @@ defmodule CF.RestApi.VideoView do
id: video.id,
hash_id: video.hash_id,
title: video.title,
- provider: video.provider,
- provider_id: video.provider_id,
+ provider: "youtube",
+ provider_id: video.youtube_id,
+ youtube_id: video.youtube_id,
+ youtube_offset: video.youtube_offset,
url: DB.Schema.Video.build_url(video),
posted_at: video.inserted_at,
speakers: render_many(video.speakers, CF.RestApi.SpeakerView, "speaker.json"),
diff --git a/apps/cf_rest_api/mix.exs b/apps/cf_rest_api/mix.exs
index cd3d27eb..4cbf4653 100644
--- a/apps/cf_rest_api/mix.exs
+++ b/apps/cf_rest_api/mix.exs
@@ -4,7 +4,7 @@ defmodule CF.RestApi.Mixfile do
def project do
[
app: :cf_rest_api,
- version: "0.9.0",
+ version: "0.9.1",
build_path: "../../_build",
compilers: [:phoenix, :gettext] ++ Mix.compilers(),
config_path: "../../config/config.exs",
diff --git a/apps/cf_rest_api/test/controllers/video_controller_test.exs b/apps/cf_rest_api/test/controllers/video_controller_test.exs
index 1198f2b1..ce3a9efd 100644
--- a/apps/cf_rest_api/test/controllers/video_controller_test.exs
+++ b/apps/cf_rest_api/test/controllers/video_controller_test.exs
@@ -18,7 +18,7 @@ defmodule CF.RestApi.VideoControllerTest do
assert Enum.count(videos) == Enum.count(response)
assert random_video.title ==
- Enum.find(response, &(&1["provider_id"] == random_video.provider_id))["title"]
+ Enum.find(response, &(&1["youtube_id"] == random_video.youtube_id))["title"]
end
test "filter on language", %{conn: conn} do
@@ -67,6 +67,6 @@ defmodule CF.RestApi.VideoControllerTest do
|> post("/videos", %{url: "https://google.fr"})
|> json_response(422)
- assert response == %{"error" => %{"url" => "Invalid URL"}}
+ assert response == %{"errors" => %{"url" => ["invalid_url"]}}
end
end
diff --git a/apps/db/lib/db_schema/video.ex b/apps/db/lib/db_schema/video.ex
index 1fad2c37..2ba5f4e7 100644
--- a/apps/db/lib/db_schema/video.ex
+++ b/apps/db/lib/db_schema/video.ex
@@ -13,31 +13,31 @@ defmodule DB.Schema.Video do
field(:title, :string)
field(:hash_id, :string)
field(:url, :string, virtual: true)
- field(:provider, :string, null: false)
- field(:provider_id, :string, null: false)
field(:language, :string, null: true)
field(:unlisted, :boolean, null: false)
field(:is_partner, :boolean, null: false)
+ # DEPRECATED
+ # field(:provider, :string)
+ # field(:provider_id, :string)
+
+ field(:youtube_id, :string)
+ field(:youtube_offset, :integer, null: false, default: 0)
+
many_to_many(:speakers, Speaker, join_through: VideoSpeaker, on_delete: :delete_all)
has_many(:statements, Statement, on_delete: :delete_all)
- timestamps()
+ timestamps(type: :utc_datetime)
end
# Define valid providers
@providers_regexs %{
- # Map a provider name to its regex, using named_captures to get the id --------------------↘️
- "youtube" =>
+ # Map a provider name to its regex, using named_captures to get the id ---------↘️
+ youtube:
~r/(?:youtube\.com\/(?:[^\/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)(?[^"&?\/ ]{11})/i
}
- # Allow for URLs like `https://anything.com/__TEST__/ID` in tests
- if Application.get_env(:db, :env) == :test do
- @providers_regexs Map.put(@providers_regexs, "__TEST__", ~r/__TEST__\/(?[^"&?\/ ]+)/)
- end
-
# Define queries
@doc """
@@ -108,28 +108,23 @@ defmodule DB.Schema.Video do
## Examples
iex> import DB.Schema.Video, only: [build_url: 1]
- iex> build_url(%{provider: "youtube", provider_id: "dQw4w9WgXcQ"})
+ iex> build_url(%{youtube_id: "dQw4w9WgXcQ"})
"https://www.youtube.com/watch?v=dQw4w9WgXcQ"
"""
- def build_url(%{provider: "youtube", provider_id: id}),
+ def build_url(%{youtube_id: id}) when not is_nil(id),
do: "https://www.youtube.com/watch?v=#{id}"
# Add a special case for building test URLs
if Application.get_env(:db, :env) == :test do
- def build_url(%{provider: "__TEST__", provider_id: id}),
+ def build_url(%{youtube_id: id}),
do: "__TEST__/#{id}"
end
@doc """
Returns overview image url for the given video
"""
- def image_url(
- _video = %__MODULE__{
- provider: "youtube",
- provider_id: youtube_id
- }
- ) do
- "https://img.youtube.com/vi/#{youtube_id}/hqdefault.jpg"
+ def image_url(_video = %__MODULE__{youtube_id: id}) when not is_nil(id) do
+ "https://img.youtube.com/vi/#{id}/hqdefault.jpg"
end
@doc """
@@ -139,11 +134,10 @@ defmodule DB.Schema.Video do
struct
|> cast(params, [:url, :title, :language])
|> validate_required([:url, :title])
- |> parse_url()
- |> validate_required([:provider, :provider_id])
+ |> parse_video_url()
|> validate_length(:title, min: 5, max: 120)
- |> unique_constraint(:videos_provider_provider_id_index)
- # Change "en-US" to "en"
+ |> unique_constraint(:videos_youtube_id_index)
+ # Change locales like "en-US" to "en"
|> update_change(:language, &hd(String.split(&1, "-")))
end
@@ -160,29 +154,26 @@ defmodule DB.Schema.Video do
end
@doc """
- Parse an URL.
- If given a changeset, fill the `provider` and `provider_id` fields or add
- an error if URL is not valid.
- If given a binary, return {provider, id} or nil if invalid.
+ Builds a changeset that allows shifting statements for all providers
## Examples
- iex> import DB.Schema.Video, only: [parse_url: 1]
- iex> parse_url ""
- nil
- iex> parse_url "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
- {"youtube", "dQw4w9WgXcQ"}
- iex> parse_url "https://www.youtube.com/watch?v="
- nil
+ iex> DB.Schema.Video.changeset_shift_offsets(%DB.Schema.Video{}, %{youtube_offset: 42})
+ #Ecto.Changeset, valid?: true>
+ """
+ def changeset_shift_offsets(struct, params \\ %{}) do
+ cast(struct, params, [:youtube_offset])
+ end
+
+ @doc """
+ Given a changeset, fill the `{provider}_id` fields or add an error if URL is not valid.
"""
- def parse_url(changeset = %Ecto.Changeset{}) do
+ def parse_video_url(changeset = %Ecto.Changeset{}) do
case changeset do
%Ecto.Changeset{valid?: true, changes: %{url: url}} ->
case parse_url(url) do
- {provider, id} ->
- changeset
- |> put_change(:provider, provider)
- |> put_change(:provider_id, id)
+ {:youtube, id} ->
+ put_change(changeset, :youtube_id, id)
_ ->
add_error(changeset, :url, "invalid_url")
@@ -193,6 +184,20 @@ defmodule DB.Schema.Video do
end
end
+ @doc """
+ Parse an URL.
+ Given a binary, return {provider, id} or nil if invalid.
+
+ ## Examples
+
+ iex> import DB.Schema.Video, only: [parse_url: 1]
+ iex> parse_url ""
+ nil
+ iex> parse_url "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
+ {:youtube, "dQw4w9WgXcQ"}
+ iex> parse_url "https://www.youtube.com/watch?v="
+ nil
+ """
def parse_url(url) when is_binary(url) do
Enum.find_value(@providers_regexs, fn {provider, regex} ->
case Regex.named_captures(regex, url) do
diff --git a/apps/db/lib/db_utils/string.ex b/apps/db/lib/db_utils/string.ex
index 9ef2e313..3dbc7452 100644
--- a/apps/db/lib/db_utils/string.ex
+++ b/apps/db/lib/db_utils/string.ex
@@ -13,6 +13,12 @@ defmodule DB.Utils.String do
iex> DB.Utils.String.trim_all_whitespaces ""
""
"""
- def trim_all_whitespaces(str),
- do: String.replace(String.trim(str), ~r/\s+/, " ")
+ def trim_all_whitespaces(nil),
+ do: nil
+
+ def trim_all_whitespaces(str) do
+ str
+ |> String.trim()
+ |> String.replace(~r/\s+/, " ")
+ end
end
diff --git a/apps/db/mix.exs b/apps/db/mix.exs
index 785944e6..5b39ab35 100644
--- a/apps/db/mix.exs
+++ b/apps/db/mix.exs
@@ -4,7 +4,7 @@ defmodule DB.Mixfile do
def project do
[
app: :db,
- version: "0.9.0",
+ version: "0.9.1",
build_path: "../../_build",
config_path: "../../config/config.exs",
deps_path: "../../deps",
diff --git a/apps/db/priv/repo/migrations/20180827123706_add_hash_id_to_videos.exs b/apps/db/priv/repo/migrations/20180827123706_add_hash_id_to_videos.exs
index 26b95b4b..781f70a7 100644
--- a/apps/db/priv/repo/migrations/20180827123706_add_hash_id_to_videos.exs
+++ b/apps/db/priv/repo/migrations/20180827123706_add_hash_id_to_videos.exs
@@ -1,28 +1,31 @@
defmodule DB.Repo.Migrations.AddHashIdToVideos do
use Ecto.Migration
+ import Ecto.Query
alias DB.Schema.Video
def up do
alter table(:videos) do
# A size of 10 allows us to go up to 100_000_000_000_000 videos
- add :hash_id, :string, size: 10
+ add(:hash_id, :string, size: 10)
end
# Create unique index on hash_id
- create unique_index(:videos, [:hash_id])
+ create(unique_index(:videos, [:hash_id]))
# Flush pending migrations to ensure column is created
flush()
# Update all existing videos with their hashIds
- DB.Repo.all(Video)
+ Video
+ |> select([v], v.id)
+ |> DB.Repo.all()
|> Enum.map(&Video.changeset_generate_hash_id/1)
|> Enum.map(&DB.Repo.update/1)
end
def down do
alter table(:videos) do
- remove :hash_id
+ remove(:hash_id)
end
end
end
diff --git a/apps/db/priv/repo/migrations/20181209205427_videos_providers_as_columns.exs b/apps/db/priv/repo/migrations/20181209205427_videos_providers_as_columns.exs
new file mode 100644
index 00000000..d620b8dc
--- /dev/null
+++ b/apps/db/priv/repo/migrations/20181209205427_videos_providers_as_columns.exs
@@ -0,0 +1,62 @@
+defmodule DB.Repo.Migrations.VideosProvidersAsColumns do
+ @moduledoc """
+ This migration changes the `Videos` schema to go from a pair
+ of {provider, provider_id} to a model where we have multiple `{provider}_id`
+ column. This will allow to store multiple sources for a single video, with
+ different offsets to ensure they're all in sync.
+ """
+
+ use Ecto.Migration
+
+ def up do
+ # Add new columns
+ alter table(:videos) do
+ add(:youtube_id, :string, null: true, length: 11)
+ add(:youtube_offset, :integer, null: false, default: 0)
+ end
+
+ flush()
+
+ # Migrate existing videos - we only have YouTube right now
+ Ecto.Adapters.SQL.query!(DB.Repo, """
+ UPDATE videos SET youtube_id = provider_id;
+ """)
+
+ flush()
+
+ # Create index
+ create(unique_index(:videos, :youtube_id))
+
+ # Remove columns
+ alter table(:videos) do
+ remove(:provider)
+ remove(:provider_id)
+ end
+ end
+
+ def down do
+ # Restore old scheme
+ alter table(:videos) do
+ add(:provider, :string)
+ add(:provider_id, :string)
+ end
+
+ flush()
+
+ # Migrate existing videos
+ Ecto.Adapters.SQL.query!(DB.Repo, """
+ UPDATE videos SET provider_id = youtube_id, provider = 'youtube';
+ """)
+
+ flush()
+
+ # Re-create index
+ create(unique_index(:videos, [:provider, :provider_id]))
+
+ # Remove columns
+ alter table(:videos) do
+ remove(:youtube_id)
+ remove(:youtube_offset)
+ end
+ end
+end
diff --git a/apps/db/test/support/factory.ex b/apps/db/test/support/factory.ex
index 0b56875f..0c19a135 100644
--- a/apps/db/test/support/factory.ex
+++ b/apps/db/test/support/factory.ex
@@ -50,8 +50,7 @@ defmodule DB.Factory do
%Video{
url: "https://www.youtube.com/watch?v=#{youtube_id}",
title: Faker.Lorem.sentence(3..8),
- provider: "youtube",
- provider_id: youtube_id,
+ youtube_id: youtube_id,
hash_id: nil,
language: Enum.random(["en", "fr", nil])
}
diff --git a/dev/_common.sh b/dev/_common.sh
deleted file mode 100644
index 1c9b852b..00000000
--- a/dev/_common.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-CF_ELIXIR_IMAGE=bitwalker/alpine-elixir-phoenix:1.6.6
-CF_DB_DEV_IMAGE=cf_dev_db
diff --git a/dev/check_gitlab_ci.sh b/dev/check_gitlab_ci.sh
deleted file mode 100755
index 93c2cd30..00000000
--- a/dev/check_gitlab_ci.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-cd -- "$(dirname $0)/.."
-
-
-echo "Converting YAML file to JSON"
-echo "============================"
-
-CONTENT=$(
- cat .gitlab-ci.yml \
- | yaml2json \
- | tr -d '\n' \
- | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))'
-)
-
-echo $CONTENT
-
-echo ""
-echo ""
-echo "Checking CI config against Gitlab's API"
-echo "======================================="
-
-curl --header "Content-Type: application/json" https://gitlab.com/api/v4/ci/lint --data "{\"content\": $CONTENT}"
\ No newline at end of file
diff --git a/dev/check_release.sh b/dev/check_release.sh
deleted file mode 100755
index b6314cc5..00000000
--- a/dev/check_release.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-#---------------------------------------------------------------------------------------------------
-# Start a docker release with dev params
-# Usage ./test_docker_release.sh
-#
-# /!\ Database must be started
-#---------------------------------------------------------------------------------------------------
-
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 app_name"
- echo "Example: $0 cf_jobs"
- exit 1
-fi
-
-
-TMP_IMAGE_NAME=cf_dev_check_release
-
-# If any command fails, exit
-set -e
-
-# Build
-cd -- "$(dirname $0)"
-docker build -t $TMP_IMAGE_NAME --build-arg APP=$1 ../
-
-# Run server
-echo "Let's test this app! =>"
-docker run -it \
- -e "CF_HOST=localhost" \
- -e "CF_SECRET_KEY_BASE=8C6FsJwjV11d+1WPUIbkEH6gB/VavJrcXWoPLujgpclfxjkLkoNFSjVU9XfeNm6s" \
- -e "CF_S3_ACCESS_KEY_ID=test" \
- -e "CF_S3_SECRET_ACCESS_KEY=test" \
- -e "CF_S3_BUCKET=test" \
- -e "CF_DB_HOSTNAME=localhost" \
- -e "CF_DB_USERNAME=postgres" \
- -e "CF_DB_PASSWORD=postgres" \
- -e "CF_DB_NAME=captain_fact_dev" \
- -e "CF_FACEBOOK_APP_ID=506726596325615" \
- -e "CF_FACEBOOK_APP_SECRET=4b320056746b8e57144c889f3baf0424" \
- -e "CF_FRONTEND_URL=http://localhost:3333" \
- -e "CF_CHROME_EXTENSION_ID=chrome-extension://lpdmcoikcclagelhlmibniibjilfifac" \
- -e "CF_BASIC_AUTH_PASSWORD=password" \
- -v "$(pwd)/../priv/secrets:/run/secrets:ro" \
- --network host \
- --rm ${TMP_IMAGE_NAME} console
diff --git a/dev/code_style.sh b/dev/code_style.sh
deleted file mode 100755
index b5dc2871..00000000
--- a/dev/code_style.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-cd -- "$(dirname $0)/.."
-source "./dev/_common.sh"
-
-docker run -it \
- --rm \
- --workdir=/app \
- --link $CF_DB_DEV_IMAGE:localhost \
- -v `pwd`:/app \
- ${CF_ELIXIR_IMAGE} \
- iex -S mix credo $@
diff --git a/dev/db_gen_migration.sh b/dev/db_gen_migration.sh
deleted file mode 100755
index 1fde2800..00000000
--- a/dev/db_gen_migration.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-if [ "$#" -lt 1 ]; then
- echo "Usage: $0 MigrationName [MigrationParams...]"
- echo "----------------------Examples--------------------------"
- echo "$0 AddLocaleToInvitationRequest"
- exit 1
-fi
-
-IMAGE=bitwalker/alpine-elixir-phoenix:1.6.0
-cd -- "$(dirname $0)/.."
-docker run -it --rm --workdir=/app/apps/db --network=host -v `pwd`:/app ${IMAGE} mix ecto.gen.migration $@ &&
- echo "Migration created in apps/db/priv/repo/migrations"
\ No newline at end of file
diff --git a/dev/db_migrate.sh b/dev/db_migrate.sh
deleted file mode 100755
index 7074f547..00000000
--- a/dev/db_migrate.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-cd -- "$(dirname $0)/.."
-source "./dev/_common.sh"
-
-docker run -it \
- --rm \
- --workdir=/app \
- --link $CF_DB_DEV_IMAGE:localhost \
- -v `pwd`:/app \
- ${CF_ELIXIR_IMAGE} \
- mix ecto.migrate
diff --git a/dev/docker_recreate_db.sh b/dev/docker_recreate_db.sh
deleted file mode 100755
index 24162b47..00000000
--- a/dev/docker_recreate_db.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-docker stop cf_dev_db
-docker rm cf_dev_db
-docker run -d --name cf_dev_db -p 5432:5432 captainfact/dev-db:latest
diff --git a/dev/get_dependencies.sh b/dev/get_dependencies.sh
deleted file mode 100755
index 145a6343..00000000
--- a/dev/get_dependencies.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-cd -- "$(dirname $0)/.."
-source "./dev/_common.sh"
-
-docker run -it \
- --rm \
- --workdir=/app \
- -v `pwd`:/app \
- ${CF_ELIXIR_IMAGE} \
- mix deps.get
diff --git a/dev/run_command.sh b/dev/run_command.sh
deleted file mode 100755
index e9b853c6..00000000
--- a/dev/run_command.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-cd -- "$(dirname $0)/.."
-source "./dev/_common.sh"
-
-if [ "$#" -lt 1 ]; then
- echo "Usage: $0 command"
- echo "----------------------Examples--------------------------"
- echo "Run test suite: $0 mix test"
- echo "Start dev server: $0 iex -S mix phx.server"
- echo "DB - Run migrations: $0 mix ecto.migrate"
- exit 1
-fi
-
-docker run -it \
- --rm \
- --workdir=/app \
- -p 4000:4000 \
- -p 4001:4001 \
- -p 4002:4002 \
- -p 4003:4003 \
- -p 4004:4004 \
- --link $CF_DB_DEV_IMAGE:localhost \
- -v `pwd`:/app \
- ${CF_ELIXIR_IMAGE} \
- $@
-
\ No newline at end of file
diff --git a/dev/start_server.sh b/dev/start_server.sh
deleted file mode 100755
index a3088428..00000000
--- a/dev/start_server.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-cd -- "$(dirname $0)/.."
-source "./dev/_common.sh"
-
-docker run -it \
- --rm \
- --workdir=/app \
- -p 4000:4000 \
- -p 4001:4001 \
- -p 4002:4002 \
- -p 4003:4003 \
- -p 4004:4004 \
- --link $CF_DB_DEV_IMAGE:localhost \
- -v `pwd`:/app \
- ${CF_ELIXIR_IMAGE} \
- iex -S mix phx.server
diff --git a/dev/test.sh b/dev/test.sh
deleted file mode 100755
index 1a597607..00000000
--- a/dev/test.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-#
-# Tests runner.
-#
-# You can test a specific subset by running ./dev/test.sh test/your_test_subpath
-#
-# Examples :
-# ./dev/test.sh test/db_schema
-# ./dev/test.sh test/captain_fact_jobs
-# ./dev/test.sh test/captain_fact_jobs/votes_test.exs
-#
-# ------------------------------------------------------------------------------
-
-cd -- "$(dirname $0)/.."
-source "./dev/_common.sh"
-
-docker run -it \
- --rm \
- --workdir=/app \
- --link $CF_DB_DEV_IMAGE:localhost \
- -v `pwd`:/app \
- ${CF_ELIXIR_IMAGE} \
- mix test $@
diff --git a/dev/watch_test.sh b/dev/watch_test.sh
deleted file mode 100755
index 3a920df6..00000000
--- a/dev/watch_test.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-#
-# Tests runner.
-#
-# You can test a specific subset by running ./dev/test.sh test/your_test_subpath
-#
-# Examples :
-# ./dev/test.sh test/db_schema
-# ./dev/test.sh test/captain_fact_jobs
-# ./dev/test.sh test/captain_fact_jobs/votes_test.exs
-#
-# ------------------------------------------------------------------------------
-
-cd -- "$(dirname $0)/.."
-source "./dev/_common.sh"
-
-docker run -it \
- --rm \
- --workdir=/app \
- --link $CF_DB_DEV_IMAGE:localhost \
- -v `pwd`:/app \
- ${CF_ELIXIR_IMAGE} \
- mix test.watch $@
diff --git a/rel/release.sh b/rel/release.sh
new file mode 100755
index 00000000..095bb963
--- /dev/null
+++ b/rel/release.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Build all releases for given tags and push them to Docker registry
+# ------------------------------------------------------------------
+
+if [ "$#" -ne 1 ]; then
+ echo "Usage: $0 environment"
+ echo "Example: $0 staging"
+ exit 1
+fi
+
+if [ -z "$DOCKER_USERNAME" ] || [ -z "$DOCKER_PASSWORD" ]; then
+ echo "[Warning] Docker credentials not provided. You must be logged in to push to registry."
+fi
+
+set -e
+
+# ---- Images names ----
+CF_API_REST_IMAGE=captainfact/rest-api:$1
+CF_API_GRAPHQL_IMAGE=captainfact/graphql-api:$1
+CF_API_ATOM_FEED=captainfact/atom-feed:$1
+CF_API_OPENGRAPH_IMAGE=captainfact/opengraph:$1
+CF_API_JOBS_IMAGE=captainfact/jobs:$1
+
+# ---- Build ----
+echo "[RELEASE] Building Apps 🔨"
+docker build --build-arg APP=cf_rest_api -t ${CF_API_REST_IMAGE} .
+docker build --build-arg APP=cf_graphql -t ${CF_API_GRAPHQL_IMAGE} .
+docker build --build-arg APP=cf_atom_feed -t ${CF_API_ATOM_FEED} .
+docker build --build-arg APP=cf_opengraph -t ${CF_API_OPENGRAPH_IMAGE} .
+docker build --build-arg APP=cf_jobs -t ${CF_API_JOBS_IMAGE} .
+
+# ---- Push release ----
+echo "[RELEASE] Pushing Apps 🚀"
+docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
+docker push $CF_API_REST_IMAGE
+docker push $CF_API_GRAPHQL_IMAGE
+docker push $CF_API_ATOM_FEED
+docker push $CF_API_OPENGRAPH_IMAGE
+docker push $CF_API_JOBS_IMAGE