diff --git a/.github/workflows/build-private-images-ghcr.yml b/.github/workflows/build-private-images-ghcr.yml index 4ccc4ac88d7f..89ed37bee13b 100644 --- a/.github/workflows/build-private-images-ghcr.yml +++ b/.github/workflows/build-private-images-ghcr.yml @@ -13,7 +13,7 @@ concurrency: jobs: build: - if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'deploy-to-staging') }} + if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'preview') }} runs-on: buildjet-16vcpu-ubuntu-2204 permissions: @@ -46,7 +46,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/build-private-images.yml b/.github/workflows/build-private-images.yml index c84177cb5910..21efe813b8a5 100644 --- a/.github/workflows/build-private-images.yml +++ b/.github/workflows/build-private-images.yml @@ -38,7 +38,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/build-public-images-ghcr.yml b/.github/workflows/build-public-images-ghcr.yml index 7794b31c211b..e5a6e9b0d99b 100644 --- a/.github/workflows/build-public-images-ghcr.yml +++ b/.github/workflows/build-public-images-ghcr.yml @@ -42,7 +42,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/build-public-images.yml b/.github/workflows/build-public-images.yml index e88c932b9fed..be13e77c9d80 100644 --- a/.github/workflows/build-public-images.yml +++ b/.github/workflows/build-public-images.yml @@ -37,7 +37,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/comment-preview-url.yml b/.github/workflows/comment-preview-url.yml new file mode 100644 index 000000000000..ca49eb1e5b58 --- /dev/null +++ b/.github/workflows/comment-preview-url.yml @@ -0,0 +1,28 @@ +name: Add preview environment URL to PR + +on: + pull_request: + types: [labeled] + +permissions: + pull-requests: write + +env: + PR_NUMBER: ${{ github.event.number }} + +jobs: + comment: + if: ${{ contains(github.event.pull_request.labels.*.name, 'preview') }} + runs-on: ubuntu-latest + steps: + - name: Comment with preview URL + uses: thollander/actions-comment-pull-request@v2.5.0 + with: + message: | +
+ + |Preview environment👷🏼‍♀️🏗️ | + |:-:| + | [PR-${{env.PR_NUMBER}}](https://pr-${{env.PR_NUMBER}}.review.plausible.io) + +
diff --git a/.tool-versions b/.tool-versions index 49e065e49863..18cb90df085f 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -erlang 26.2.1 -elixir 1.16.0-otp-26 +erlang 27.0 +elixir 1.17.1-otp-27 nodejs 21.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 559d559fa281..75f271eeb96a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,15 +4,24 @@ All notable changes to this project will be documented in this file. ## Unreleased ### Added +- Icons for browsers plausible/analytics#4239 +- Automatic custom property selection in the dashboard Properties report +- Add `does_not_contain` filter support to dashboard ### Removed +- Deprecate `ECTO_IPV6` and `ECTO_CH_IPV6` env vars in CE plausible/analytics#4245 ### Changed - Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200 +- Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245 +- `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead. +- `bounce_rate` metric now returns 0 instead of null for event:page breakdown when page has never been entry page. ### Fixed +- Fix access to Stats API feature in CE plausible/analytics#4244 + ## v2.1.1 - 2024-06-06 ### Added diff --git a/Dockerfile b/Dockerfile index 26e75aa97d76..835c326bd746 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # platform specific, it makes sense to build it in the docker #### Builder -FROM hexpm/elixir:1.16.0-erlang-26.2.1-alpine-3.18.4 as buildcontainer +FROM hexpm/elixir:1.17.1-erlang-27.0-alpine-3.18.6 as buildcontainer ARG MIX_ENV=ce @@ -28,7 +28,7 @@ COPY mix.lock ./ COPY config ./config RUN mix local.hex --force && \ mix local.rebar --force && \ - mix deps.get --only prod && \ + mix deps.get --only ${MIX_ENV} && \ mix deps.compile COPY assets/package.json assets/package-lock.json ./assets/ @@ -54,7 +54,7 @@ COPY rel rel RUN mix release plausible # Main Docker Image -FROM alpine:3.18.4 +FROM alpine:3.18.6 LABEL maintainer="plausible.io " ARG BUILD_METADATA={} diff --git a/assets/js/dashboard.js b/assets/js/dashboard.js index a172b05264d4..693e17986649 100644 --- a/assets/js/dashboard.js +++ b/assets/js/dashboard.js @@ -31,7 +31,8 @@ if (container) { background: container.dataset.background, isDbip: container.dataset.isDbip === 'true', flags: JSON.parse(container.dataset.flags), - validIntervalsByPeriod: JSON.parse(container.dataset.validIntervalsByPeriod) + validIntervalsByPeriod: JSON.parse(container.dataset.validIntervalsByPeriod), + shared: !!container.dataset.sharedLinkAuth, } const loggedIn = container.dataset.loggedIn === 'true' diff --git a/assets/js/dashboard/components/combobox.js b/assets/js/dashboard/components/combobox.js index a5268f75e9dd..b6f85098b2e0 100644 --- a/assets/js/dashboard/components/combobox.js +++ b/assets/js/dashboard/components/combobox.js @@ -37,7 +37,7 @@ function optionId(index) { export default function PlausibleCombobox(props) { const [options, setOptions] = useState([]) - const [loading, setLoading] = useState(false) + const [isLoading, setLoading] = useState(false) const [isOpen, setOpen] = useState(false) const [input, setInput] = useState('') const [highlightedIndex, setHighlightedIndex] = useState(0) @@ -45,6 +45,8 @@ export default function PlausibleCombobox(props) { const containerRef = useRef(null) const listRef = useRef(null) + const loading = isLoading || !!props.forceLoading + const visibleOptions = [...options] if (props.freeChoice && input.length > 0 && options.every(option => option.value !== input)) { visibleOptions.push({value: input, label: input, freeChoice: true}) diff --git a/assets/js/dashboard/components/filter-operator-selector.js b/assets/js/dashboard/components/filter-operator-selector.js index a33edd80b797..5086e6abf6c5 100644 --- a/assets/js/dashboard/components/filter-operator-selector.js +++ b/assets/js/dashboard/components/filter-operator-selector.js @@ -1,6 +1,6 @@ import React, { Fragment } from "react"; -import { FILTER_OPERATIONS } from "../util/filters"; +import { FILTER_OPERATIONS, FILTER_OPERATIONS_DISPLAY_NAMES } from "../util/filters"; import { Menu, Transition } from "@headlessui/react"; import { ChevronDownIcon } from '@heroicons/react/20/solid' import { isFreeChoiceFilter, supportsIsNot } from "../util/filters"; @@ -9,20 +9,20 @@ import classNames from "classnames"; export default function FilterOperatorSelector(props) { const filterName = props.forFilter - function renderTypeItem(type, shouldDisplay) { + function renderTypeItem(operation, shouldDisplay) { return ( shouldDisplay && ( {({ active }) => ( props.onSelect(type)} + onClick={() => props.onSelect(operation)} className={classNames("cursor-pointer block px-4 py-2 text-sm", { "bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-100": active, "text-gray-700 dark:text-gray-200": !active } )} > - {type} + {FILTER_OPERATIONS_DISPLAY_NAMES[operation]} )} @@ -40,8 +40,8 @@ export default function FilterOperatorSelector(props) { {({ open }) => ( <>
- - {props.selectedType} + + {FILTER_OPERATIONS_DISPLAY_NAMES[props.selectedType]}
@@ -58,12 +58,13 @@ export default function FilterOperatorSelector(props) { >
{renderTypeItem(FILTER_OPERATIONS.is, true)} {renderTypeItem(FILTER_OPERATIONS.isNot, supportsIsNot(filterName))} {renderTypeItem(FILTER_OPERATIONS.contains, isFreeChoiceFilter(filterName))} + {renderTypeItem(FILTER_OPERATIONS.does_not_contain, isFreeChoiceFilter(filterName))}
diff --git a/assets/js/dashboard/custom-hooks.js b/assets/js/dashboard/custom-hooks.js new file mode 100644 index 000000000000..adb955f15c30 --- /dev/null +++ b/assets/js/dashboard/custom-hooks.js @@ -0,0 +1,15 @@ +import { useEffect, useRef } from 'react'; + +// A custom hook that behaves like `useEffect`, but +// the function does not run on the initial render. +export function useMountedEffect(fn, deps) { + const mounted = useRef(false) + + useEffect(() => { + if (mounted.current) { + fn() + } else { + mounted.current = true + } + }, deps) +} \ No newline at end of file diff --git a/assets/js/dashboard/filters.js b/assets/js/dashboard/filters.js index c675973c44d4..0b26a158dc34 100644 --- a/assets/js/dashboard/filters.js +++ b/assets/js/dashboard/filters.js @@ -13,7 +13,8 @@ import { formattedFilters, EVENT_PROPS_PREFIX, getPropertyKeyFromFilterKey, - getLabel + getLabel, + FILTER_OPERATIONS_DISPLAY_NAMES } from "./util/filters" const WRAPSTATE = { unwrapped: 0, waiting: 1, wrapped: 2 } @@ -41,10 +42,10 @@ function filterText(query, [operation, filterKey, clauses]) { const formattedFilter = formattedFilters[filterKey] if (formattedFilter) { - return <>{formattedFilter} {operation} {clauses.map((value) => {getLabel(query.labels, filterKey, value)}).reduce((prev, curr) => [prev, ' or ', curr])} + return <>{formattedFilter} {FILTER_OPERATIONS_DISPLAY_NAMES[operation]} {clauses.map((value) => {getLabel(query.labels, filterKey, value)}).reduce((prev, curr) => [prev, ' or ', curr])} } else if (filterKey.startsWith(EVENT_PROPS_PREFIX)) { const propKey = getPropertyKeyFromFilterKey(filterKey) - return <>Property {propKey} {operation} {clauses.map((label) => {label}).reduce((prev, curr) => [prev, ' or ', curr])} + return <>Property {propKey} {FILTER_OPERATIONS_DISPLAY_NAMES[operation]} {clauses.map((label) => {label}).reduce((prev, curr) => [prev, ' or ', curr])} } throw new Error(`Unknown filter: ${filterKey}`) @@ -83,7 +84,7 @@ function filterDropdownOption(site, option) { {({ active }) => ( filterDropdownOption(site, option)) @@ -127,9 +128,11 @@ function Filters(props) { const [viewport, setViewport] = useState(1080) useEffect(() => { + handleResize() + window.addEventListener('resize', handleResize, false) document.addEventListener('keyup', handleKeyup) - + return () => { window.removeEventListener('resize', handleResize, false) document.removeEventListener("keyup", handleKeyup) @@ -190,7 +193,7 @@ function Filters(props) { title={`Edit filter: ${formattedFilters[type]}`} className="flex w-full h-full items-center py-2 pl-3" to={{ - pathname: `/${encodeURIComponent(site.domain)}/filter/${FILTER_GROUP_TO_MODAL_TYPE[type]}`, + pathname: `/filter/${FILTER_GROUP_TO_MODAL_TYPE[type]}`, search: window.location.search }} > @@ -228,7 +231,7 @@ function Filters(props) { } function trackFilterMenu() { - window.plausible && window.plausible('Filter Menu: Open', {u: `${window.location.protocol}//${window.location.hostname}/:dashboard`}) + window.plausible && window.plausible('Filter Menu: Open', { u: `${window.location.protocol}//${window.location.hostname}/:dashboard` }) } function renderDropDown() { diff --git a/assets/js/dashboard/index.js b/assets/js/dashboard/index.js index c13c233ece2c..ca755c344344 100644 --- a/assets/js/dashboard/index.js +++ b/assets/js/dashboard/index.js @@ -1,6 +1,7 @@ import React, { useEffect, useState } from 'react'; import { withRouter } from 'react-router-dom' +import { useMountedEffect } from './custom-hooks'; import Historical from './historical' import Realtime from './realtime' import {parseQuery} from './query' @@ -23,7 +24,7 @@ function Dashboard(props) { } }, []) - useEffect(() => { + useMountedEffect(() => { api.cancelAll() setQuery(parseQuery(location.search, site)) updateLastLoadTimestamp() diff --git a/assets/js/dashboard/router.js b/assets/js/dashboard/router.js index 4605c9fd15c6..8f591f62249d 100644 --- a/assets/js/dashboard/router.js +++ b/assets/js/dashboard/router.js @@ -28,50 +28,50 @@ function ScrollToTop() { export default function Router({ site, loggedIn, currentUserRole }) { return ( - - + + - + - + - + - + - + - + - - + + - + - + - + - + - + - + ); } diff --git a/assets/js/dashboard/stats/behaviours/conversions.js b/assets/js/dashboard/stats/behaviours/conversions.js index c474b5786714..6161b9457f14 100644 --- a/assets/js/dashboard/stats/behaviours/conversions.js +++ b/assets/js/dashboard/stats/behaviours/conversions.js @@ -34,7 +34,7 @@ export default function Conversions(props) { BUILD_EXTRA && { name: 'total_revenue', label: 'Revenue', hiddenOnMobile: true }, BUILD_EXTRA && { name: 'average_revenue', label: 'Average', hiddenOnMobile: true } ]} - detailsLink={url.sitePath(site, '/conversions')} + detailsLink={url.sitePath('conversions')} maybeHideDetails={true} query={query} color="bg-red-50" diff --git a/assets/js/dashboard/stats/behaviours/goal-conversions.js b/assets/js/dashboard/stats/behaviours/goal-conversions.js index 2d7b5263cc9a..0996e33d25cb 100644 --- a/assets/js/dashboard/stats/behaviours/goal-conversions.js +++ b/assets/js/dashboard/stats/behaviours/goal-conversions.js @@ -7,10 +7,11 @@ import * as api from "../../api" import { EVENT_PROPS_PREFIX, getGoalFilter } from "../../util/filters" export const SPECIAL_GOALS = { - '404': {title: '404 Pages', prop: 'path'}, - 'Outbound Link: Click': {title: 'Outbound Links', prop: 'url'}, - 'Cloaked Link: Click': {title: 'Cloaked Links', prop: 'url'}, - 'File Download': {title: 'File Downloads', prop: 'url'} + '404': { title: '404 Pages', prop: 'path' }, + 'Outbound Link: Click': { title: 'Outbound Links', prop: 'url' }, + 'Cloaked Link: Click': { title: 'Cloaked Links', prop: 'url' }, + 'File Download': { title: 'File Downloads', prop: 'url' }, + 'WP Search Queries': { title: 'WordPress Search Queries', prop: 'search_query' }, } function getSpecialGoal(query) { @@ -59,11 +60,11 @@ function SpecialPropBreakdown(props) { getFilterFor={getFilterFor} keyLabel={prop} metrics={[ - {name: 'visitors', label: 'Visitors', plot: true}, - {name: 'events', label: 'Events', hiddenOnMobile: true}, + { name: 'visitors', label: 'Visitors', plot: true }, + { name: 'events', label: 'Events', hiddenOnMobile: true }, CR_METRIC ]} - detailsLink={url.sitePath(site, `/custom-prop-values/${prop}`)} + detailsLink={url.sitePath(`custom-prop-values/${prop}`)} externalLinkDest={externalLinkDest()} maybeHideDetails={true} query={query} @@ -74,7 +75,7 @@ function SpecialPropBreakdown(props) { } export default function GoalConversions(props) { - const {site, query, afterFetchData} = props + const { site, query, afterFetchData } = props const specialGoal = getSpecialGoal(query) if (specialGoal) { diff --git a/assets/js/dashboard/stats/behaviours/index.js b/assets/js/dashboard/stats/behaviours/index.js index ce45dbbc6ea2..f79e0ed044fa 100644 --- a/assets/js/dashboard/stats/behaviours/index.js +++ b/assets/js/dashboard/stats/behaviours/index.js @@ -125,7 +125,7 @@ export default function Behaviours(props) { leaveFrom="opacity-100 scale-100" leaveTo="opacity-0 scale-95" > - +
{funnelNames.map((funnelName) => { return ( @@ -182,7 +182,7 @@ export default function Behaviours(props) { function renderConversions() { if (site.hasGoals) { - return + return } else if (adminAccess) { return ( @@ -213,9 +213,9 @@ export default function Behaviours(props) { let callToAction if (site.funnelsAvailable) { - callToAction = {action: 'Set up funnels', link: `/${encodeURIComponent(site.domain)}/settings/funnels`} + callToAction = { action: 'Set up funnels', link: `/${encodeURIComponent(site.domain)}/settings/funnels` } } else { - callToAction = {action: 'Upgrade', link: '/billing/choose-plan'} + callToAction = { action: 'Upgrade', link: '/billing/choose-plan' } } return ( @@ -234,14 +234,14 @@ export default function Behaviours(props) { function renderProps() { if (site.hasProps && site.propsAvailable) { - return + return } else if (adminAccess) { let callToAction if (site.propsAvailable) { - callToAction = {action: 'Set up props', link: `/${encodeURIComponent(site.domain)}/settings/properties`} + callToAction = { action: 'Set up props', link: `/${encodeURIComponent(site.domain)}/settings/properties` } } else { - callToAction = {action: 'Upgrade', link: '/billing/choose-plan'} + callToAction = { action: 'Upgrade', link: '/billing/choose-plan' } } return ( @@ -337,11 +337,11 @@ export default function Behaviours(props) { function renderImportedQueryUnsupportedWarning() { if (mode === CONVERSIONS) { - return + return } else if (mode === PROPS) { - return + return } else { - return + return } } @@ -354,7 +354,7 @@ export default function Behaviours(props) {

{sectionTitle() + (isRealtime() ? ' (last 30min)' : '')}

- { renderImportedQueryUnsupportedWarning()} + {renderImportedQueryUnsupportedWarning()}
{tabs()} diff --git a/assets/js/dashboard/stats/behaviours/props.js b/assets/js/dashboard/stats/behaviours/props.js index 17ebe5e245c8..4986b215df77 100644 --- a/assets/js/dashboard/stats/behaviours/props.js +++ b/assets/js/dashboard/stats/behaviours/props.js @@ -1,11 +1,12 @@ -import React, { useCallback, useState } from "react" -import ListReport from "../reports/list"; +import React, { useCallback, useEffect, useState } from "react" +import ListReport, { MIN_HEIGHT } from "../reports/list"; import Combobox from '../../components/combobox' import * as api from '../../api' import * as url from '../../util/url' import { CR_METRIC, PERCENTAGE_METRIC } from "../reports/metrics"; import * as storage from "../../util/storage"; -import { getFiltersByKeyPrefix, EVENT_PROPS_PREFIX, getPropertyKeyFromFilterKey, getGoalFilter, FILTER_OPERATIONS, hasGoalFilter } from "../../util/filters" +import { EVENT_PROPS_PREFIX, getGoalFilter, FILTER_OPERATIONS, hasGoalFilter } from "../../util/filters" +import classNames from "classnames"; export default function Properties(props) { @@ -16,7 +17,8 @@ export default function Properties(props) { return `${goal}__prop_key__${site.domain}` } - const [propKey, setPropKey] = useState(choosePropKey()) + const [propKey, setPropKey] = useState(null) + const [propKeyLoading, setPropKeyLoading] = useState(true) function singleGoalFilterApplied() { const goalFilter = getGoalFilter(query) @@ -28,15 +30,26 @@ export default function Properties(props) { } } - function choosePropKey() { - const propFilters = getFiltersByKeyPrefix(query, EVENT_PROPS_PREFIX) - if (propFilters.length > 0) { - const [_operation, filterKey, _clauses] = propFilters[0] - return getPropertyKeyFromFilterKey(filterKey) - } else { - return getPropKeyFromStorage() - } - } + useEffect(() => { + setPropKeyLoading(true) + setPropKey(null) + + fetchPropKeyOptions()("").then((propKeys) => { + const propKeyValues = propKeys.map(entry => entry.value) + + if (propKeyValues.length > 0) { + const storedPropKey = getPropKeyFromStorage() + + if (propKeyValues.includes(storedPropKey)) { + setPropKey(storedPropKey) + } else { + setPropKey(propKeys[0].value) + } + } + + setPropKeyLoading(false) + }) + }, [query]) function getPropKeyFromStorage() { if (singleGoalFilterApplied()) { @@ -85,7 +98,7 @@ export default function Properties(props) { BUILD_EXTRA && { name: 'total_revenue', label: 'Revenue', hiddenOnMobile: true }, BUILD_EXTRA && { name: 'average_revenue', label: 'Average', hiddenOnMobile: true } ]} - detailsLink={url.sitePath(site, `/custom-prop-values/${propKey}`)} + detailsLink={`/custom-prop-values/${propKey}`} maybeHideDetails={true} query={query} color="bg-red-50" @@ -99,13 +112,19 @@ export default function Properties(props) { filter: ["is", `${EVENT_PROPS_PREFIX}${propKey}`, [listItem.name]] }) + const comboboxDisabled = !propKeyLoading && !propKey + const comboboxPlaceholder = comboboxDisabled ? 'No custom properties found' : '' const comboboxValues = propKey ? [{ value: propKey, label: propKey }] : [] - const boxClass = 'pl-2 pr-8 py-1 bg-transparent dark:text-gray-300 rounded-md shadow-sm border border-gray-300 dark:border-gray-500' + const boxClass = classNames('pl-2 pr-8 py-1 bg-transparent dark:text-gray-300 rounded-md shadow-sm border border-gray-300 dark:border-gray-500', { + 'pointer-events-none': comboboxDisabled + }) + + const COMBOBOX_HEIGHT = 40 return ( -
-
- +
+
+
{propKey && renderBreakdown()}
diff --git a/assets/js/dashboard/stats/devices/index.js b/assets/js/dashboard/stats/devices/index.js index cb668f7871c2..ddb2bcd7ca86 100644 --- a/assets/js/dashboard/stats/devices/index.js +++ b/assets/js/dashboard/stats/devices/index.js @@ -7,6 +7,38 @@ import * as url from '../../util/url' import { VISITORS_METRIC, PERCENTAGE_METRIC, maybeWithCR } from '../reports/metrics'; import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning'; +// Icons copied from https://github.com/alrra/browser-logos +const BROWSER_ICONS = { + 'Chrome': 'chrome.svg', + 'Safari': 'safari.png', + 'Firefox': 'firefox.svg', + 'Microsoft Edge': 'edge.svg', + 'Vivaldi': 'vivaldi.svg', + 'Opera': 'opera.svg', + 'Samsung Browser': 'samsung-internet.svg', + 'Chromium': 'chromium.svg', + 'UC Browser': 'uc.svg', + 'Yandex Browser': 'yandex.png', // Only PNG available in browser-logos + // Logos underneath this line are not available in browser-logos. Grabbed from random places on the internets. + 'DuckDuckGo Privacy Browser': 'duckduckgo.svg', + 'MIUI Browser': 'miui.webp', + 'Huawei Browser Mobile': 'huawei.png', + 'QQ Browser': 'qq.png', + 'Ecosia': 'ecosia.png', + 'vivo Browser': 'vivo.png' +} + +function browserIconFor(browser) { + const filename = BROWSER_ICONS[browser] || 'fallback.svg' + + return ( + + ) +} + function Browsers({ query, site, afterFetchData }) { function fetchData() { return api.get(url.apiPath(site, '/browsers'), query) @@ -19,6 +51,10 @@ function Browsers({ query, site, afterFetchData }) { } } + function renderIcon(listItem) { + return browserIconFor(listItem.name) + } + return ( ) } @@ -34,6 +71,15 @@ function Browsers({ query, site, afterFetchData }) { function BrowserVersions({ query, site, afterFetchData }) { function fetchData() { return api.get(url.apiPath(site, '/browser-versions'), query) + .then(res => { + return {...res, results: res.results.map((row => { + return {...row, name: `${row.browser} ${row.name}`, version: row.name} + }))} + }) + } + + function renderIcon(listItem) { + return browserIconFor(listItem.browser) } function getFilterFor(listItem) { @@ -42,7 +88,7 @@ function BrowserVersions({ query, site, afterFetchData }) { } return { prefix: 'browser_version', - filter: ["is", "browser_version", [listItem['name']]] + filter: ["is", "browser_version", [listItem.version]] } } @@ -53,6 +99,7 @@ function BrowserVersions({ query, site, afterFetchData }) { getFilterFor={getFilterFor} keyLabel="Browser version" metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)} + renderIcon={renderIcon} query={query} /> ) @@ -117,7 +164,9 @@ function ScreenSizes({ query, site, afterFetchData }) { } function renderIcon(screenSize) { - return iconFor(screenSize.name) + return ( + {iconFor(screenSize.name)} + ) } function getFilterFor(listItem) { diff --git a/assets/js/dashboard/stats/graph/stats-export.js b/assets/js/dashboard/stats/graph/stats-export.js index 6716b2eb1eb6..0c3613a81767 100644 --- a/assets/js/dashboard/stats/graph/stats-export.js +++ b/assets/js/dashboard/stats/graph/stats-export.js @@ -2,7 +2,7 @@ import React, { useState } from "react" import * as api from '../../api' import { getCurrentInterval } from "./interval-picker" -export default function StatsExport({site, query}) { +export default function StatsExport({ site, query }) { const [exporting, setExporting] = useState(false) function startExport() { @@ -50,4 +50,4 @@ export default function StatsExport({site, query}) { {!exporting && renderExportLink()}
) -} \ No newline at end of file +} diff --git a/assets/js/dashboard/stats/locations/index.js b/assets/js/dashboard/stats/locations/index.js index 7a229f449d0f..23f4a4bc2ded 100644 --- a/assets/js/dashboard/stats/locations/index.js +++ b/assets/js/dashboard/stats/locations/index.js @@ -4,108 +4,108 @@ import * as storage from '../../util/storage' import CountriesMap from './map' import * as api from '../../api' -import {apiPath, sitePath} from '../../util/url' +import { apiPath, sitePath } from '../../util/url' import ListReport from '../reports/list' import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics'; import { getFiltersByKeyPrefix } from '../../util/filters'; import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning'; -function Countries({query, site, onClick, afterFetchData}) { - function fetchData() { - return api.get(apiPath(site, '/countries'), query, { limit: 9 }) - } - - function renderIcon(country) { - return {country.flag} - } - - function getFilterFor(listItem) { - return { - prefix: "country", - filter: ["is", "country", [listItem['code']]], - labels: { [listItem['code']]: listItem['name'] } - } - } - - return ( - - ) +function Countries({ query, site, onClick, afterFetchData }) { + function fetchData() { + return api.get(apiPath(site, '/countries'), query, { limit: 9 }) + } + + function renderIcon(country) { + return {country.flag} + } + + function getFilterFor(listItem) { + return { + prefix: "country", + filter: ["is", "country", [listItem['code']]], + labels: { [listItem['code']]: listItem['name'] } + } + } + + return ( + + ) } -function Regions({query, site, onClick, afterFetchData}) { - function fetchData() { - return api.get(apiPath(site, '/regions'), query, {limit: 9}) - } - - function renderIcon(region) { - return {region.country_flag} - } - - function getFilterFor(listItem) { - return { - prefix: "region", - filter: ["is", "region", [listItem['code']]], - labels: { [listItem['code']]: listItem['name'] } - } - } - - return ( - - ) +function Regions({ query, site, onClick, afterFetchData }) { + function fetchData() { + return api.get(apiPath(site, '/regions'), query, { limit: 9 }) + } + + function renderIcon(region) { + return {region.country_flag} + } + + function getFilterFor(listItem) { + return { + prefix: "region", + filter: ["is", "region", [listItem['code']]], + labels: { [listItem['code']]: listItem['name'] } + } + } + + return ( + + ) } -function Cities({query, site, afterFetchData}) { - function fetchData() { - return api.get(apiPath(site, '/cities'), query, {limit: 9}) - } - - function renderIcon(city) { - return {city.country_flag} - } - - function getFilterFor(listItem) { - return { - prefix: "city", - filter: ["is", "city", [listItem['code']]], - labels: { [listItem['code']]: listItem['name'] } - } - } - - return ( - - ) +function Cities({ query, site, afterFetchData }) { + function fetchData() { + return api.get(apiPath(site, '/cities'), query, { limit: 9 }) + } + + function renderIcon(city) { + return {city.country_flag} + } + + function getFilterFor(listItem) { + return { + prefix: "city", + filter: ["is", "city", [listItem['code']]], + labels: { [listItem['code']]: listItem['name'] } + } + } + + return ( + + ) } @@ -117,116 +117,116 @@ const labelFor = { export default class Locations extends React.Component { constructor(props) { - super(props) - this.onCountryFilter = this.onCountryFilter.bind(this) - this.onRegionFilter = this.onRegionFilter.bind(this) - this.afterFetchData = this.afterFetchData.bind(this) - this.tabKey = `geoTab__${ props.site.domain}` - const storedTab = storage.getItem(this.tabKey) - this.state = { - mode: storedTab || 'map', - loading: true, - skipImportedReason: null - } - } - - componentDidUpdate(prevProps, prevState) { - const isRemovingFilter = (filterName) => { - return getFiltersByKeyPrefix(prevProps.query, filterName).length > 0 && - getFiltersByKeyPrefix(this.props.query, filterName).length == 0 - } - - if (this.state.mode === 'cities' && isRemovingFilter('region')) { - this.setMode('regions')() - } - - if (this.state.mode === 'regions' && isRemovingFilter('country')) { - this.setMode(this.countriesRestoreMode || 'countries')() - } - - if (this.props.query !== prevProps.query || this.state.mode !== prevState.mode) { - this.setState({loading: true}) - } - } - - setMode(mode) { - return () => { - storage.setItem(this.tabKey, mode) - this.setState({mode}) - } - } - - onCountryFilter(mode) { - return () => { - this.countriesRestoreMode = mode - this.setMode('regions')() - } - } - - onRegionFilter() { - this.setMode('cities')() - } - - afterFetchData(apiResponse) { - this.setState({loading: false, skipImportedReason: apiResponse.skip_imported_reason}) - } + super(props) + this.onCountryFilter = this.onCountryFilter.bind(this) + this.onRegionFilter = this.onRegionFilter.bind(this) + this.afterFetchData = this.afterFetchData.bind(this) + this.tabKey = `geoTab__${props.site.domain}` + const storedTab = storage.getItem(this.tabKey) + this.state = { + mode: storedTab || 'map', + loading: true, + skipImportedReason: null + } + } + + componentDidUpdate(prevProps, prevState) { + const isRemovingFilter = (filterName) => { + return getFiltersByKeyPrefix(prevProps.query, filterName).length > 0 && + getFiltersByKeyPrefix(this.props.query, filterName).length == 0 + } + + if (this.state.mode === 'cities' && isRemovingFilter('region')) { + this.setMode('regions')() + } + + if (this.state.mode === 'regions' && isRemovingFilter('country')) { + this.setMode(this.countriesRestoreMode || 'countries')() + } + + if (this.props.query !== prevProps.query || this.state.mode !== prevState.mode) { + this.setState({ loading: true }) + } + } + + setMode(mode) { + return () => { + storage.setItem(this.tabKey, mode) + this.setState({ mode }) + } + } + + onCountryFilter(mode) { + return () => { + this.countriesRestoreMode = mode + this.setMode('regions')() + } + } + + onRegionFilter() { + this.setMode('cities')() + } + + afterFetchData(apiResponse) { + this.setState({ loading: false, skipImportedReason: apiResponse.skip_imported_reason }) + } renderContent() { - switch(this.state.mode) { - case "cities": - return - case "regions": - return - case "countries": - return - case "map": - default: - return - } - } + switch (this.state.mode) { + case "cities": + return + case "regions": + return + case "countries": + return + case "map": + default: + return + } + } renderPill(name, mode) { - const isActive = this.state.mode === mode - - if (isActive) { - return ( - - ) - } - - return ( - - ) - } + const isActive = this.state.mode === mode + + if (isActive) { + return ( + + ) + } + + return ( + + ) + } render() { - return ( -
-
-
-

- {labelFor[this.state.mode] || 'Locations'} -

- -
-
- { this.renderPill('Map', 'map') } - { this.renderPill('Countries', 'countries') } - { this.renderPill('Regions', 'regions') } - { this.renderPill('Cities', 'cities') } -
-
- {this.renderContent()} -
- ) - } + return ( +
+
+
+

+ {labelFor[this.state.mode] || 'Locations'} +

+ +
+
+ {this.renderPill('Map', 'map')} + {this.renderPill('Countries', 'countries')} + {this.renderPill('Regions', 'regions')} + {this.renderPill('Cities', 'cities')} +
+
+ {this.renderContent()} +
+ ) + } } diff --git a/assets/js/dashboard/stats/locations/map.js b/assets/js/dashboard/stats/locations/map.js index 4006607c80ea..2cd09c8a340b 100644 --- a/assets/js/dashboard/stats/locations/map.js +++ b/assets/js/dashboard/stats/locations/map.js @@ -28,7 +28,7 @@ class Countries extends React.Component { componentDidUpdate(prevProps) { if (this.props.query !== prevProps.query) { // eslint-disable-next-line react/no-did-update-set-state - this.setState({loading: true, countries: null}) + this.setState({ loading: true, countries: null }) this.fetchCountries().then(this.drawMap) } } @@ -49,19 +49,19 @@ class Countries extends React.Component { getDataset() { const dataset = {}; - var onlyValues = this.state.countries.map(function(obj){ return obj.visitors }); + var onlyValues = this.state.countries.map(function(obj) { return obj.visitors }); var maxValue = Math.max.apply(null, onlyValues); // eslint-disable-next-line no-undef const paletteScale = d3.scale.linear() - .domain([0,maxValue]) + .domain([0, maxValue]) .range([ this.state.darkTheme ? "#2e3954" : "#f3ebff", this.state.darkTheme ? "#6366f1" : "#a779e9" ]) - this.state.countries.forEach(function(item){ - dataset[item.alpha_3] = {numberOfThings: item.visitors, fillColor: paletteScale(item.visitors)}; + this.state.countries.forEach(function(item) { + dataset[item.alpha_3] = { numberOfThings: item.visitors, fillColor: paletteScale(item.visitors) }; }); return dataset @@ -69,18 +69,18 @@ class Countries extends React.Component { updateCountries() { this.fetchCountries().then(() => { - this.map.updateChoropleth(this.getDataset(), {reset: true}) + this.map.updateChoropleth(this.getDataset(), { reset: true }) }) } fetchCountries() { - return api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/countries`, this.props.query, {limit: 300}) + return api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/countries`, this.props.query, { limit: 300 }) .then((response) => { if (this.props.afterFetchData) { this.props.afterFetchData(response) } - this.setState({loading: false, countries: response.results}) + this.setState({ loading: false, countries: response.results }) }) } @@ -154,9 +154,9 @@ class Countries extends React.Component { if (this.state.countries) { return ( <> -
- - { this.geolocationDbNotice() } +
+ + {this.geolocationDbNotice()} ) } @@ -167,9 +167,9 @@ class Countries extends React.Component { render() { return ( - { this.state.loading &&
} + {this.state.loading &&
} - { this.renderBody() } + {this.renderBody()}
) diff --git a/assets/js/dashboard/stats/modals/conversions.js b/assets/js/dashboard/stats/modals/conversions.js index 1a245918cc2f..38efbfa7d032 100644 --- a/assets/js/dashboard/stats/modals/conversions.js +++ b/assets/js/dashboard/stats/modals/conversions.js @@ -69,7 +69,7 @@ function ConversionsModal(props) { {listItem.name} @@ -117,7 +117,7 @@ function ConversionsModal(props) { } return ( - + {renderBody()} {loading && renderLoading()} {!loading && moreResultsAvailable && renderLoadMore()} diff --git a/assets/js/dashboard/stats/modals/entry-pages.js b/assets/js/dashboard/stats/modals/entry-pages.js index 3c7004f42184..04007c6fd68c 100644 --- a/assets/js/dashboard/stats/modals/entry-pages.js +++ b/assets/js/dashboard/stats/modals/entry-pages.js @@ -81,7 +81,7 @@ class EntryPagesModal extends React.Component { + {this.renderBody()} {this.renderLoading()} diff --git a/assets/js/dashboard/stats/modals/exit-pages.js b/assets/js/dashboard/stats/modals/exit-pages.js index 8dc82bec08ff..25be36b3ac23 100644 --- a/assets/js/dashboard/stats/modals/exit-pages.js +++ b/assets/js/dashboard/stats/modals/exit-pages.js @@ -4,7 +4,7 @@ import { withRouter } from 'react-router-dom' import Modal from './modal' import * as api from '../../api' -import numberFormatter, {percentageFormatter} from '../../util/number-formatter' +import numberFormatter, { percentageFormatter } from '../../util/number-formatter' import { parseQuery } from '../../query' import { trimURL, updatedQuery } from '../../util/url' import { hasGoalFilter, replaceFilterByPrefix } from "../../util/filters"; @@ -62,7 +62,7 @@ class ExitPagesModal extends React.Component { + {this.renderBody()} {this.renderLoading()} diff --git a/assets/js/dashboard/stats/modals/filter-modal-group.js b/assets/js/dashboard/stats/modals/filter-modal-group.js index b8eee128e4ba..3b1722f143e6 100644 --- a/assets/js/dashboard/stats/modals/filter-modal-group.js +++ b/assets/js/dashboard/stats/modals/filter-modal-group.js @@ -17,7 +17,6 @@ export default function FilterModalGroup({ () => Object.entries(filterState).filter(([_, filter]) => getFilterGroup(filter) == filterGroup).map(([id, filter]) => ({ id, filter })), [filterGroup, filterState] ) - const disabledOptions = useMemo( () => (filterGroup == 'props') ? rows.map(({ filter }) => ({ value: getPropertyKeyFromFilterKey(filter[1]) })) : null, [filterGroup, rows] @@ -28,7 +27,7 @@ export default function FilterModalGroup({ return ( <> -
+
{showTitle && (
{formattedFilters[filterGroup]}
)} {rows.map(({ id, filter }) => filterGroup === 'props' ? ( @@ -55,7 +54,7 @@ export default function FilterModalGroup({ )}
{showAddRow && ( -
+
onAddRow(filterGroup)}> + Add another diff --git a/assets/js/dashboard/stats/modals/filter-modal-props-row.js b/assets/js/dashboard/stats/modals/filter-modal-props-row.js index 7f08be1f1a1c..09fb3e77ab07 100644 --- a/assets/js/dashboard/stats/modals/filter-modal-props-row.js +++ b/assets/js/dashboard/stats/modals/filter-modal-props-row.js @@ -33,7 +33,7 @@ export default function FilterModalPropsRow({ } function fetchPropValueOptions(input) { - if (operation === FILTER_OPERATIONS.contains) {return Promise.resolve([])} + if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.does_not_contain].includes(operation)) {return Promise.resolve([])} return fetchSuggestions(apiPath(site, `/suggestions/prop_value`), query, input, [ FILTER_OPERATIONS.isNot, filterKey, ['(none)'] ]) diff --git a/assets/js/dashboard/stats/modals/filter-modal-row.js b/assets/js/dashboard/stats/modals/filter-modal-row.js index 09c28781520f..dc4f85555f45 100644 --- a/assets/js/dashboard/stats/modals/filter-modal-row.js +++ b/assets/js/dashboard/stats/modals/filter-modal-row.js @@ -32,7 +32,7 @@ export default function FilterModalRow({ } function fetchOptions(input) { - if (operation === FILTER_OPERATIONS.contains) { + if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.does_not_contain].includes(operation)) { return Promise.resolve([]) } @@ -43,14 +43,14 @@ export default function FilterModalRow({ return (
-
+
onUpdate([newOperation, filterKey, clauses], labels)} selectedType={operation} />
-
+
{ - const filterState = {...prevState.filterState} + const filterState = { ...prevState.filterState } delete filterState[id] return { filterState } }) @@ -133,7 +133,7 @@ class FilterModal extends React.Component { render() { return ( - +

Filter by {formatFilterGroup(this.state.modalType)}

diff --git a/assets/js/dashboard/stats/modals/google-keywords.js b/assets/js/dashboard/stats/modals/google-keywords.js index 39ba7f3b8fca..4dfd45972679 100644 --- a/assets/js/dashboard/stats/modals/google-keywords.js +++ b/assets/js/dashboard/stats/modals/google-keywords.js @@ -4,7 +4,7 @@ import { Link, withRouter } from 'react-router-dom' import Modal from './modal' import * as api from '../../api' import numberFormatter, { percentageFormatter } from '../../util/number-formatter' -import {parseQuery} from '../../query' +import { parseQuery } from '../../query' import RocketIcon from './rocket-icon' class GoogleKeywordsModal extends React.Component { @@ -17,7 +17,7 @@ class GoogleKeywordsModal extends React.Component { } componentDidMount() { - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, {limit: 100}) + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, { limit: 100 }) .then((res) => this.setState({ loading: false, searchTerms: res.search_terms, @@ -99,7 +99,7 @@ class GoogleKeywordsModal extends React.Component {
- { this.renderKeywords() } + {this.renderKeywords()}
) @@ -108,8 +108,8 @@ class GoogleKeywordsModal extends React.Component { render() { return ( - - { this.renderBody() } + + {this.renderBody()} ) } diff --git a/assets/js/dashboard/stats/modals/modal.js b/assets/js/dashboard/stats/modals/modal.js index 4b031f3600dc..a6cbbabded39 100644 --- a/assets/js/dashboard/stats/modals/modal.js +++ b/assets/js/dashboard/stats/modals/modal.js @@ -57,7 +57,7 @@ class Modal extends React.Component { } close() { - this.props.history.push(`/${encodeURIComponent(this.props.site.domain)}${this.props.location.search}`) + this.props.history.push(`/${this.props.location.search}`) } /** diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.js index 643938352fde..d564a2689c54 100644 --- a/assets/js/dashboard/stats/modals/pages.js +++ b/assets/js/dashboard/stats/modals/pages.js @@ -65,7 +65,7 @@ class PagesModal extends React.Component { -

Top Pages

+

Top Pages hi

@@ -141,7 +141,7 @@ class PagesModal extends React.Component { render() { return ( - + {this.renderBody()} {this.renderLoading()} diff --git a/assets/js/dashboard/stats/modals/props.js b/assets/js/dashboard/stats/modals/props.js index 0fcd1c04f562..64fe66a97b57 100644 --- a/assets/js/dashboard/stats/modals/props.js +++ b/assets/js/dashboard/stats/modals/props.js @@ -25,7 +25,8 @@ const Money = maybeRequire().default function PropsModal(props) { const site = props.site const query = parseQuery(props.location.search, site) - const propKey = props.location.pathname.split('/').pop() + + const propKey = props.location.pathname.split('/').filter(i => i).pop() const [loading, setLoading] = useState(true) const [moreResultsAvailable, setMoreResultsAvailable] = useState(false) @@ -71,7 +72,7 @@ function PropsModal(props) { {url.trimURL(listItem.name, 30)} @@ -125,7 +126,7 @@ function PropsModal(props) { } return ( - + {renderBody()} {loading && renderLoading()} {!loading && moreResultsAvailable && renderLoadMore()} diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js index 49d33c547690..ff1e4caf5bf2 100644 --- a/assets/js/dashboard/stats/modals/referrer-drilldown.js +++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js @@ -3,8 +3,8 @@ import { Link, withRouter } from 'react-router-dom' import Modal from './modal' import * as api from '../../api' -import numberFormatter, {durationFormatter} from '../../util/number-formatter' -import {parseQuery} from '../../query' +import numberFormatter, { durationFormatter } from '../../util/number-formatter' +import { parseQuery } from '../../query' import { updatedQuery } from "../../util/url"; import { hasGoalFilter, replaceFilterByPrefix } from "../../util/filters"; @@ -20,8 +20,8 @@ class ReferrerDrilldownModal extends React.Component { componentDidMount() { const detailed = this.showExtra() - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/${this.props.match.params.referrer}`, this.state.query, {limit: 100, detailed}) - .then((response) => this.setState({loading: false, referrers: response.results})) + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/${this.props.match.params.referrer}`, this.state.query, { limit: 100, detailed }) + .then((response) => this.setState({ loading: false, referrers: response.results })) } showExtra() { @@ -45,7 +45,7 @@ class ReferrerDrilldownModal extends React.Component { } formatBounceRate(ref) { - if (typeof(ref.bounce_rate) === 'number') { + if (typeof (ref.bounce_rate) === 'number') { return ref.bounce_rate + '%' } else { return '-' @@ -53,7 +53,7 @@ class ReferrerDrilldownModal extends React.Component { } formatDuration(referrer) { - if (typeof(referrer.visit_duration) === 'number') { + if (typeof (referrer.visit_duration) === 'number') { return durationFormatter(referrer.visit_duration) } else { return '-' @@ -77,12 +77,12 @@ class ReferrerDrilldownModal extends React.Component { {referrer.name} - { this.renderExternalLink(referrer.name) } + {this.renderExternalLink(referrer.name)} ) } @@ -91,13 +91,13 @@ class ReferrerDrilldownModal extends React.Component { return ( - { this.renderReferrerName(referrer) } + {this.renderReferrerName(referrer)} - {this.showConversionRate() && {numberFormatter(referrer.total_visitors)} } + {this.showConversionRate() && {numberFormatter(referrer.total_visitors)}} {numberFormatter(referrer.visitors)} - {this.showExtra() && {this.formatBounceRate(referrer)} } - {this.showExtra() && {this.formatDuration(referrer)} } - {this.showConversionRate() && {referrer.conversion_rate}% } + {this.showExtra() && {this.formatBounceRate(referrer)}} + {this.showExtra() && {this.formatDuration(referrer)}} + {this.showConversionRate() && {referrer.conversion_rate}%} ) } @@ -126,7 +126,7 @@ class ReferrerDrilldownModal extends React.Component { - { this.state.referrers.map(this.renderReferrer.bind(this)) } + {this.state.referrers.map(this.renderReferrer.bind(this))}
@@ -137,8 +137,8 @@ class ReferrerDrilldownModal extends React.Component { render() { return ( - - { this.renderBody() } + + {this.renderBody()} ) } diff --git a/assets/js/dashboard/stats/modals/sources.js b/assets/js/dashboard/stats/modals/sources.js index 2f272f19e017..8e9efa068dc9 100644 --- a/assets/js/dashboard/stats/modals/sources.js +++ b/assets/js/dashboard/stats/modals/sources.js @@ -152,7 +152,7 @@ class SourcesModal extends React.Component { render() { return ( - +

{this.title()}

diff --git a/assets/js/dashboard/stats/modals/table.js b/assets/js/dashboard/stats/modals/table.js index 20eea170c3bd..fc4861776a10 100644 --- a/assets/js/dashboard/stats/modals/table.js +++ b/assets/js/dashboard/stats/modals/table.js @@ -4,7 +4,7 @@ import { Link, withRouter } from 'react-router-dom' import Modal from './modal' import * as api from '../../api' import numberFormatter from '../../util/number-formatter' -import {parseQuery} from '../../query' +import { parseQuery } from '../../query' import { cleanLabels, hasGoalFilter, replaceFilterByPrefix } from "../../util/filters"; import { updatedQuery } from "../../util/url"; @@ -18,8 +18,8 @@ class ModalTable extends React.Component { } componentDidMount() { - api.get(this.props.endpoint, this.state.query, {limit: 100}) - .then((response) => this.setState({loading: false, list: response.results})) + api.get(this.props.endpoint, this.state.query, { limit: 100 }) + .then((response) => this.setState({ loading: false, list: response.results })) } showConversionRate() { @@ -56,7 +56,7 @@ class ModalTable extends React.Component { className="hover:underline" to={{ search: updatedQuery({ filters, labels }), - pathname: `/${encodeURIComponent(this.props.site.domain)}` + pathname: `/` }} > {this.props.renderIcon && this.props.renderIcon(tableItem)} @@ -97,7 +97,7 @@ class ModalTable extends React.Component { - { this.state.list.map(this.renderTableItem.bind(this)) } + {this.state.list.map(this.renderTableItem.bind(this))} @@ -110,8 +110,8 @@ class ModalTable extends React.Component { render() { return ( - - { this.renderBody() } + + {this.renderBody()} ) } diff --git a/assets/js/dashboard/stats/more-link.js b/assets/js/dashboard/stats/more-link.js index 95dfdf284e8b..3940cbbc5050 100644 --- a/assets/js/dashboard/stats/more-link.js +++ b/assets/js/dashboard/stats/more-link.js @@ -5,7 +5,7 @@ function detailsIcon() { return ( 0) { return (
- { detailsIcon() } + {detailsIcon()} DETAILS
diff --git a/assets/js/dashboard/stats/pages/index.js b/assets/js/dashboard/stats/pages/index.js index 991605fd4e09..ee76a8bdced1 100644 --- a/assets/js/dashboard/stats/pages/index.js +++ b/assets/js/dashboard/stats/pages/index.js @@ -30,7 +30,7 @@ function EntryPages({ query, site, afterFetchData }) { getFilterFor={getFilterFor} keyLabel="Entry page" metrics={maybeWithCR([{ ...VISITORS_METRIC, label: 'Unique Entrances' }], query)} - detailsLink={url.sitePath(site, '/entry-pages')} + detailsLink={url.sitePath('entry-pages')} query={query} externalLinkDest={externalLinkDest} color="bg-orange-50" @@ -61,7 +61,7 @@ function ExitPages({ query, site, afterFetchData }) { getFilterFor={getFilterFor} keyLabel="Exit page" metrics={maybeWithCR([{ ...VISITORS_METRIC, label: "Unique Exits" }], query)} - detailsLink={url.sitePath(site, '/exit-pages')} + detailsLink={url.sitePath('exit-pages')} query={query} externalLinkDest={externalLinkDest} color="bg-orange-50" @@ -92,7 +92,7 @@ function TopPages({ query, site, afterFetchData }) { getFilterFor={getFilterFor} keyLabel="Page" metrics={maybeWithCR([VISITORS_METRIC], query)} - detailsLink={url.sitePath(site, '/pages')} + detailsLink={url.sitePath('pages')} query={query} externalLinkDest={externalLinkDest} color="bg-orange-50" @@ -107,7 +107,7 @@ const labelFor = { } export default function Pages(props) { - const {site, query} = props + const { site, query } = props const tabKey = `pageTab__${site.domain}` const storedTab = storage.getItem(tabKey) const [mode, setMode] = useState(storedTab || 'pages') diff --git a/assets/js/dashboard/stats/reports/list.js b/assets/js/dashboard/stats/reports/list.js index 52fe138db5b7..ed869ccf82af 100644 --- a/assets/js/dashboard/stats/reports/list.js +++ b/assets/js/dashboard/stats/reports/list.js @@ -11,14 +11,14 @@ import classNames from 'classnames' import { trimURL, updatedQuery } from '../../util/url' import { cleanLabels, hasGoalFilter, replaceFilterByPrefix } from '../../util/filters' const MAX_ITEMS = 9 -const MIN_HEIGHT = 380 +export const MIN_HEIGHT = 380 const ROW_HEIGHT = 32 const ROW_GAP_HEIGHT = 4 const DATA_CONTAINER_HEIGHT = (ROW_HEIGHT + ROW_GAP_HEIGHT) * (MAX_ITEMS - 1) + ROW_HEIGHT const COL_MIN_WIDTH = 70 function FilterLink({ filterQuery, onClick, children }) { - const className = classNames('max-w-max w-full flex md:overflow-hidden', { + const className = classNames('max-w-max w-full flex items-center md:overflow-hidden', { 'hover:underline': !!filterQuery }) @@ -276,11 +276,7 @@ export default function ListReport(props) { function maybeRenderIconFor(listItem) { if (props.renderIcon) { - return ( - - {props.renderIcon(listItem)} - - ) + return props.renderIcon(listItem) } } diff --git a/assets/js/dashboard/stats/sources/referrer-list.js b/assets/js/dashboard/stats/sources/referrer-list.js index f0069e8578df..3ebc1af6381d 100644 --- a/assets/js/dashboard/stats/sources/referrer-list.js +++ b/assets/js/dashboard/stats/sources/referrer-list.js @@ -5,14 +5,14 @@ import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics' import ListReport from '../reports/list' import ImportedQueryUnsupportedWarning from '../../stats/imported-query-unsupported-warning' -export default function Referrers({source, site, query}) { +export default function Referrers({ source, site, query }) { const [skipImportedReason, setSkipImportedReason] = useState(null) const [loading, setLoading] = useState(true) useEffect(() => setLoading(true), [query]) function fetchReferrers() { - return api.get(url.apiPath(site, `/referrers/${encodeURIComponent(source)}`), query, {limit: 9}) + return api.get(url.apiPath(site, `/referrers/${encodeURIComponent(source)}`), query, { limit: 9 }) } function afterFetchReferrers(apiResponse) { @@ -48,7 +48,7 @@ export default function Referrers({source, site, query}) {

Top Referrers

- +
- { - this.setState({ loading: false, searchTerms: [], notConfigured: true, error: true, isAdmin: error.payload.is_admin }) - } + })).catch((error) => { + this.setState({ loading: false, searchTerms: [], notConfigured: true, error: true, isAdmin: error.payload.is_admin }) + } ) } @@ -59,7 +58,7 @@ export default class SearchTerms extends React.Component { > - { term.name } + {term.name} @@ -69,7 +68,7 @@ export default class SearchTerms extends React.Component { } renderList() { - if (this.state.unsupportedFilters) { + if (this.state.unsupportedFilters) { return (
@@ -81,10 +80,10 @@ export default class SearchTerms extends React.Component {
- This site is not connected to Search Console so we cannot show the search terms - {this.state.isAdmin && this.state.error && <>

Please click below to connect your Search Console account.

} + This site is not connected to Search Console so we cannot show the search terms + {this.state.isAdmin && this.state.error && <>

Please click below to connect your Search Console account.

}
- {this.state.isAdmin && Connect with Google } + {this.state.isAdmin && Connect with Google}
) } else if (this.state.searchTerms.length > 0) { @@ -114,8 +113,8 @@ export default class SearchTerms extends React.Component { return (

Search Terms

- { this.renderList() } - + {this.renderList()} +
) } @@ -124,10 +123,10 @@ export default class SearchTerms extends React.Component { render() { return (
- { this.state.loading &&
} + {this.state.loading &&
} - { this.renderContent() } + {this.renderContent()}
diff --git a/assets/js/dashboard/stats/sources/source-list.js b/assets/js/dashboard/stats/sources/source-list.js index f832ab17ba13..806dfc8195a3 100644 --- a/assets/js/dashboard/stats/sources/source-list.js +++ b/assets/js/dashboard/stats/sources/source-list.js @@ -36,7 +36,7 @@ function AllSources(props) { return ( ) } @@ -48,7 +48,7 @@ function AllSources(props) { getFilterFor={getFilterFor} keyLabel="Source" metrics={maybeWithCR([VISITORS_METRIC], query)} - detailsLink={url.sitePath(site, '/sources')} + detailsLink={url.sitePath('sources')} renderIcon={renderIcon} query={query} color="bg-blue-50" @@ -78,7 +78,7 @@ function UTMSources(props) { getFilterFor={getFilterFor} keyLabel={utmTag.label} metrics={maybeWithCR([VISITORS_METRIC], query)} - detailsLink={url.sitePath(site, utmTag.endpoint)} + detailsLink={url.sitePath(utmTag.endpoint)} query={query} color="bg-blue-50" /> @@ -178,7 +178,7 @@ export default function SourceList(props) {

Top Sources

- +
{renderTabs()}
diff --git a/assets/js/dashboard/util/filters.js b/assets/js/dashboard/util/filters.js index 27367e260da6..26523ae62f26 100644 --- a/assets/js/dashboard/util/filters.js +++ b/assets/js/dashboard/util/filters.js @@ -23,17 +23,26 @@ export const NO_CONTAINS_OPERATOR = new Set(['goal', 'screen'].concat(FILTER_MOD export const EVENT_PROPS_PREFIX = "props:" export const FILTER_OPERATIONS = { + is: 'is', isNot: 'is_not', contains: 'contains', - is: 'is' + does_not_contain: 'does_not_contain' }; -export const OPERATION_PREFIX = { +export const FILTER_OPERATIONS_DISPLAY_NAMES = { + [FILTER_OPERATIONS.is]: 'is', + [FILTER_OPERATIONS.isNot]: 'is not', + [FILTER_OPERATIONS.contains]: 'contains', + [FILTER_OPERATIONS.does_not_contain]: 'does not contain' +} + +const OPERATION_PREFIX = { [FILTER_OPERATIONS.isNot]: '!', [FILTER_OPERATIONS.contains]: '~', [FILTER_OPERATIONS.is]: '' }; + export function supportsIsNot(filterName) { return !['goal', 'prop_key'].includes(filterName) } @@ -53,17 +62,6 @@ try { const ESCAPED_PIPE = '\\|' -function escapeFilterValue(value) { - return value.replaceAll(NON_ESCAPED_PIPE_REGEX, ESCAPED_PIPE) -} - -function toFilterQuery(type, clauses) { - const prefix = OPERATION_PREFIX[type]; - const result = clauses.map(clause => escapeFilterValue(clause.toString().trim())).join('|') - return prefix + result; -} - - export function getLabel(labels, filterKey, value) { if (['country', 'region', 'city'].includes(filterKey)) { return labels[value] @@ -141,19 +139,19 @@ export function cleanLabels(filters, labels, mergedFilterKey, mergedLabels) { return result } +const EVENT_FILTER_KEYS = new Set(["name", "page", "goal", "hostname"]) -// :TODO: New schema for filters in the BE export function serializeApiFilters(filters) { - const cleaned = {} - filters.forEach(([operation, filterKey, clauses]) => { - if (filterKey.startsWith(EVENT_PROPS_PREFIX)) { - cleaned.props ||= {} - cleaned.props[getPropertyKeyFromFilterKey(filterKey)] = toFilterQuery(operation, clauses) - } else { - cleaned[filterKey] = toFilterQuery(operation, clauses) + const apiFilters = filters.map(([operation, filterKey, clauses]) => { + let apiFilterKey = `visit:${filterKey}` + if (filterKey.startsWith(EVENT_PROPS_PREFIX) || EVENT_FILTER_KEYS.has(filterKey)) { + apiFilterKey = `event:${filterKey}` } + clauses = clauses.map((value) => value.toString()) + return [operation, apiFilterKey, clauses] }) - return JSON.stringify(cleaned) + + return JSON.stringify(apiFilters) } export function fetchSuggestions(apiPath, query, input, additionalFilter) { diff --git a/assets/js/dashboard/util/url.js b/assets/js/dashboard/util/url.js index e9349e0d343d..f201c6fe0b22 100644 --- a/assets/js/dashboard/util/url.js +++ b/assets/js/dashboard/util/url.js @@ -1,15 +1,11 @@ import JsonURL from '@jsonurl/jsonurl' export function apiPath(site, path = '') { - return `/api/stats/${encodeURIComponent(site.domain)}${path}` + return `/api/stats/${encodeURIComponent(site.domain)}${path}/` } -export function siteBasePath(site, path = '') { - return `/${encodeURIComponent(site.domain)}${path}` -} - -export function sitePath(site, path = '') { - return siteBasePath(site, path) + window.location.search +export function sitePath(path = '') { + return (path.startsWith('/') ? path : '/' + path) + window.location.search } export function setQuery(key, value) { diff --git a/config/config.exs b/config/config.exs index 8d248d091822..7a610cfb2795 100644 --- a/config/config.exs +++ b/config/config.exs @@ -38,7 +38,7 @@ config :tailwind, config :ua_inspector, database_path: "priv/ua_inspector", - remote_release: "6.3.0" + remote_release: "6.3.2" config :ref_inspector, database_path: "priv/ref_inspector" diff --git a/config/runtime.exs b/config/runtime.exs index fd1eee218524..170f39933d45 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -319,7 +319,18 @@ config :plausible, PlausibleWeb.Endpoint, websocket_url: websocket_url, secure_cookie: secure_cookie -maybe_ipv6 = if System.get_env("ECTO_IPV6"), do: [:inet6], else: [] +maybe_ipv6 = + if get_var_from_path_or_env(config_dir, "ECTO_IPV6") do + if config_env() in [:ce, :ce_dev, :ce_test] do + Logger.warning( + "ECTO_IPV6 is no longer necessary as all TCP connections now try IPv6 automatically with IPv4 fallback" + ) + end + + [:inet6] + else + [] + end db_cacertfile = get_var_from_path_or_env(config_dir, "DATABASE_CACERTFILE", CAStore.file_path()) @@ -368,6 +379,12 @@ maybe_ch_ipv6 = get_var_from_path_or_env(config_dir, "ECTO_CH_IPV6", "false") |> String.to_existing_atom() +if maybe_ch_ipv6 && config_env() in [:ce, :ce_dev, :ce_test] do + Logger.warning( + "ECTO_CH_IPV6 is no longer necessary as all TCP connections now try IPv6 automatically with IPv4 fallback" + ) +end + ch_cacertfile = get_var_from_path_or_env(config_dir, "CLICKHOUSE_CACERTFILE") ch_transport_opts = [ @@ -606,6 +623,7 @@ if config_env() in [:dev, :staging, :prod, :test] do ecto_repo: Plausible.Repo, router: PlausibleWeb.Router, admin_title: "Plausible Admin", + extensions: [Plausible.CrmExtensions], resources: [ auth: [ resources: [ diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex index 8c4115c0aae3..7443a1758d37 100644 --- a/extra/lib/plausible/stats/goal/revenue.ex +++ b/extra/lib/plausible/stats/goal/revenue.ex @@ -15,14 +15,20 @@ defmodule Plausible.Stats.Goal.Revenue do def total_revenue_query() do dynamic( [e], - fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + selected_as( + fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount), + :total_revenue + ) ) end def average_revenue_query() do dynamic( [e], - fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + selected_as( + fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount), + :average_revenue + ) ) end @@ -40,8 +46,7 @@ defmodule Plausible.Stats.Goal.Revenue do def get_revenue_tracking_currency(site, query, metrics) do goal_filters = case Query.get_filter(query, "event:goal") do - [:is, "event:goal", {_, goal_name}] -> [goal_name] - [:member, "event:goal", list] -> Enum.map(list, fn {_, goal_name} -> goal_name end) + [:is, "event:goal", list] -> Enum.map(list, fn {_, goal_name} -> goal_name end) _ -> [] end diff --git a/extra/lib/plausible_web/plugins/api/views/funnel.ex b/extra/lib/plausible_web/plugins/api/views/funnel.ex index 7547d829394f..8ee9a67b55ca 100644 --- a/extra/lib/plausible_web/plugins/api/views/funnel.ex +++ b/extra/lib/plausible_web/plugins/api/views/funnel.ex @@ -11,7 +11,8 @@ defmodule PlausibleWeb.Plugins.API.Views.Funnel do conn: conn }) do %{ - funnels: render_many(funnels, __MODULE__, "funnel.json", authorized_site: site), + funnels: + render_many(funnels, __MODULE__, "funnel.json", authorized_site: site, as: :funnel), meta: render_metadata_links(metadata, :plugins_api_funnels_url, :index, conn.query_params) } end @@ -22,7 +23,8 @@ defmodule PlausibleWeb.Plugins.API.Views.Funnel do conn: conn }) do %{ - funnels: render_many(funnels, __MODULE__, "funnel.json", authorized_site: site), + funnels: + render_many(funnels, __MODULE__, "funnel.json", authorized_site: site, as: :funnel), meta: render_metadata_links(%{}, :plugins_api_funnels_url, :index, conn.query_params) } end @@ -42,7 +44,8 @@ defmodule PlausibleWeb.Plugins.API.Views.Funnel do id: funnel.id, steps: render_many(goals, PlausibleWeb.Plugins.API.Views.Goal, "goal.json", - authorized_site: site + authorized_site: site, + as: :goal ) } } diff --git a/fixture/http_mocks/google_analytics_import#1.json b/fixture/http_mocks/google_analytics_import#1.json index fc55a033116c..180659ed6e1e 100644 --- a/fixture/http_mocks/google_analytics_import#1.json +++ b/fixture/http_mocks/google_analytics_import#1.json @@ -48,7 +48,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": null, "viewId": "54297898" } @@ -170139,7 +170139,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": null, "viewId": "54297898" } @@ -229240,7 +229240,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": null, "viewId": "54297898" } @@ -323594,7 +323594,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": null, "viewId": "54297898" } @@ -483689,7 +483689,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": "10000", "viewId": "54297898" } @@ -521554,7 +521554,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": "10000", "viewId": "54297898" } @@ -540444,7 +540444,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": null, "viewId": "54297898" } @@ -560672,7 +560672,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": null, "viewId": "54297898" } @@ -700763,7 +700763,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": null, "viewId": "54297898" } @@ -729150,7 +729150,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": null, "viewId": "54297898" } @@ -785186,7 +785186,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": "10000", "viewId": "54297898" } @@ -805974,7 +805974,7 @@ "sortOrder": "DESCENDING" } ], - "pageSize": 7500, + "pageSize": 100000, "pageToken": null, "viewId": "54297898" } diff --git a/lib/mix/tasks/pull_sandbox_subscription.ex b/lib/mix/tasks/pull_sandbox_subscription.ex index d2824c869ebf..438b36951685 100644 --- a/lib/mix/tasks/pull_sandbox_subscription.ex +++ b/lib/mix/tasks/pull_sandbox_subscription.ex @@ -48,6 +48,7 @@ defmodule Mix.Tasks.PullSandboxSubscription do update_url: res["update_url"], user_id: user.id, status: res["state"], + last_bill_date: res["last_payment"]["date"], next_bill_date: res["next_payment"]["date"], next_bill_amount: res["next_payment"]["amount"] |> to_string(), currency_code: res["next_payment"]["currency"] diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 97d20b6e58fd..7f28ee7fa73d 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -8,6 +8,7 @@ defmodule Plausible.Application do def start(_type, _args) do on_ee(do: Plausible.License.ensure_valid_license()) + on_ce(do: :inet_db.set_tcp_module(:happy_tcp)) children = [ Plausible.Cache.Stats, @@ -24,6 +25,10 @@ defmodule Plausible.Application do Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer), Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer), ReferrerBlocklist, + Plausible.Cache.Adapter.child_spec(:customer_currency, :cache_customer_currency, + ttl_check_interval: :timer.minutes(5), + global_ttl: :timer.minutes(60) + ), Plausible.Cache.Adapter.child_spec(:user_agents, :cache_user_agents, ttl_check_interval: :timer.seconds(5), global_ttl: :timer.minutes(60) diff --git a/lib/plausible/auth/auth.ex b/lib/plausible/auth/auth.ex index 4eaf5df78ad7..026e9fc5d4cb 100644 --- a/lib/plausible/auth/auth.ex +++ b/lib/plausible/auth/auth.ex @@ -77,7 +77,7 @@ defmodule Plausible.Auth do end @spec create_api_key(Auth.User.t(), String.t(), String.t()) :: - {:ok, Auth.ApiKey.t()} | {:error, Ecto.Changeset.t()} + {:ok, Auth.ApiKey.t()} | {:error, Ecto.Changeset.t() | :upgrade_required} def create_api_key(user, name, key) do params = %{name: name, user_id: user.id, key: key} changeset = Auth.ApiKey.changeset(%Auth.ApiKey{}, params) diff --git a/lib/plausible/auth/user_admin.ex b/lib/plausible/auth/user_admin.ex index dd9bf98fb2de..8301d18a1fe9 100644 --- a/lib/plausible/auth/user_admin.ex +++ b/lib/plausible/auth/user_admin.ex @@ -36,7 +36,6 @@ defmodule Plausible.Auth.UserAdmin do trial_expiry_date: %{name: "Trial expiry", value: &format_date(&1.trial_expiry_date)}, subscription_plan: %{value: &subscription_plan/1}, subscription_status: %{value: &subscription_status/1}, - usage: %{value: &usage_link/1}, grace_period: %{value: &grace_period_status/1}, accept_traffic_until: %{ name: "Accept traffic until", @@ -110,12 +109,7 @@ defmodule Plausible.Auth.UserAdmin do quota = PlausibleWeb.AuthView.subscription_quota(user.subscription) interval = PlausibleWeb.AuthView.subscription_interval(user.subscription) - manage_url = - Plausible.Billing.PaddleApi.vendors_domain() <> - "/subscriptions/customers/manage/" <> - user.subscription.paddle_subscription_id - - {:safe, ~s(#{quota} \(#{interval}\))} + {:safe, ~s(#{quota} \(#{interval}\))} else "--" end @@ -124,7 +118,13 @@ defmodule Plausible.Auth.UserAdmin do defp subscription_status(user) do cond do user.subscription -> - PlausibleWeb.AuthView.present_subscription_status(user.subscription.status) + status_str = PlausibleWeb.AuthView.present_subscription_status(user.subscription.status) + + if user.subscription.paddle_subscription_id do + {:safe, ~s(#{status_str})} + else + status_str + end Plausible.Users.on_trial?(user) -> "On trial" @@ -134,13 +134,9 @@ defmodule Plausible.Auth.UserAdmin do end end - on_ee do - defp usage_link(user) do - path = PlausibleWeb.Router.Helpers.admin_path(PlausibleWeb.Endpoint, :usage, user.id) - {:safe, ~s(Usage)} - end - else - defp usage_link(_), do: nil + defp manage_url(%{paddle_subscription_id: paddle_id} = _subscription) do + Plausible.Billing.PaddleApi.vendors_domain() <> + "/subscriptions/customers/manage/" <> paddle_id end defp format_date(nil), do: "--" diff --git a/lib/plausible/billing/billing.ex b/lib/plausible/billing/billing.ex index 10488656adb3..0bb29e5bcc9b 100644 --- a/lib/plausible/billing/billing.ex +++ b/lib/plausible/billing/billing.ex @@ -137,7 +137,22 @@ defmodule Plausible.Billing do defp handle_subscription_updated(params) do subscription = Repo.get_by(Subscription, paddle_subscription_id: params["subscription_id"]) - if subscription do + # In a situation where the subscription is paused and a payment succeeds, we + # get notified of two "subscription_updated" webhook alerts from Paddle at the + # same time. + # + # * one with an `old_status` of "paused", and a `status` of "past_due" + # * the other with an `old_status` of "past_due", and a `status` of "active" + # + # https://developer.paddle.com/classic/guides/zg9joji1mzu0mduy-payment-failures + # + # Relying on the time when the webhooks are sent has caused issues where + # subscriptions have ended up `past_due` after a successful payment. Therefore, + # we're now explicitly ignoring the first webhook (with the update that's not + # relevant to us). + irrelevant? = params["old_status"] == "paused" && params["status"] == "past_due" + + if subscription && not irrelevant? do subscription |> Subscription.changeset(format_subscription(params)) |> Repo.update!() diff --git a/lib/plausible/billing/enterprise_plan_admin.ex b/lib/plausible/billing/enterprise_plan_admin.ex index bab7375305bb..b88b1944b24d 100644 --- a/lib/plausible/billing/enterprise_plan_admin.ex +++ b/lib/plausible/billing/enterprise_plan_admin.ex @@ -1,6 +1,15 @@ defmodule Plausible.Billing.EnterprisePlanAdmin do use Plausible.Repo + @numeric_fields [ + "user_id", + "paddle_plan_id", + "monthly_pageview_limit", + "site_limit", + "team_member_limit", + "hourly_api_request_limit" + ] + def search_fields(_schema) do [ :paddle_plan_id, @@ -40,8 +49,42 @@ defmodule Plausible.Billing.EnterprisePlanAdmin do defp get_user_email(plan), do: plan.user.email + def create_changeset(schema, attrs) do + attrs = sanitize_attrs(attrs) + Plausible.Billing.EnterprisePlan.changeset(schema, attrs) + end + def update_changeset(enterprise_plan, attrs) do - attrs = Map.put_new(attrs, "features", []) + attrs = + attrs + |> Map.put_new("features", []) + |> sanitize_attrs() + Plausible.Billing.EnterprisePlan.changeset(enterprise_plan, attrs) end + + defp sanitize_attrs(attrs) do + attrs + |> Enum.map(&clear_attr/1) + |> Enum.reject(&(&1 == "")) + |> Map.new() + end + + defp clear_attr({key, value}) when key in @numeric_fields do + value = + value + |> to_string() + |> String.replace(~r/[^0-9-]/, "") + |> String.trim() + + {key, value} + end + + defp clear_attr({key, value}) when is_binary(value) do + {key, String.trim(value)} + end + + defp clear_attr(other) do + other + end end diff --git a/lib/plausible/billing/feature.ex b/lib/plausible/billing/feature.ex index f6bed68bdbd7..4339b6f7b5bb 100644 --- a/lib/plausible/billing/feature.ex +++ b/lib/plausible/billing/feature.ex @@ -132,7 +132,7 @@ defmodule Plausible.Billing.Feature do def check_availability(%Plausible.Auth.User{} = user) do cond do free?() -> :ok - __MODULE__ in Quota.allowed_features_for(user) -> :ok + __MODULE__ in Quota.Limits.allowed_features_for(user) -> :ok true -> {:error, :upgrade_required} end end @@ -202,33 +202,38 @@ defmodule Plausible.Billing.Feature.StatsAPI do name: :stats_api, display_name: "Stats API" - @impl true - @doc """ - Checks whether the user has access to Stats API or not. - - Before the business tier, users who had not yet started their trial had - access to Stats API. With the business tier work, access is blocked and they - must either start their trial or subscribe to a plan. This is common when a - site owner invites a new user. In such cases, using the owner's API key is - recommended. - """ - def check_availability(%Plausible.Auth.User{} = user) do - user = Plausible.Users.with_subscription(user) - unlimited_trial? = is_nil(user.trial_expiry_date) - subscription? = Plausible.Billing.Subscriptions.active?(user.subscription) - - pre_business_tier_account? = - Timex.before?(user.inserted_at, Plausible.Billing.Plans.business_tier_launch()) - - cond do - !subscription? && unlimited_trial? && pre_business_tier_account? -> - :ok - - !subscription? && unlimited_trial? && !pre_business_tier_account? -> - {:error, :upgrade_required} - - true -> - super(user) + if Plausible.ee?() do + @impl true + @doc """ + Checks whether the user has access to Stats API or not. + + Before the business tier, users who had not yet started their trial had + access to Stats API. With the business tier work, access is blocked and they + must either start their trial or subscribe to a plan. This is common when a + site owner invites a new user. In such cases, using the owner's API key is + recommended. + """ + def check_availability(%Plausible.Auth.User{} = user) do + user = Plausible.Users.with_subscription(user) + unlimited_trial? = is_nil(user.trial_expiry_date) + subscription? = Plausible.Billing.Subscriptions.active?(user.subscription) + + pre_business_tier_account? = + Timex.before?(user.inserted_at, Plausible.Billing.Plans.business_tier_launch()) + + cond do + !subscription? && unlimited_trial? && pre_business_tier_account? -> + :ok + + !subscription? && unlimited_trial? && !pre_business_tier_account? -> + {:error, :upgrade_required} + + true -> + super(user) + end end + else + @impl true + def check_availability(_user), do: :ok end end diff --git a/lib/plausible/billing/paddle_api.ex b/lib/plausible/billing/paddle_api.ex index 4fb4558c76c0..a4ebffc8ee18 100644 --- a/lib/plausible/billing/paddle_api.ex +++ b/lib/plausible/billing/paddle_api.ex @@ -108,7 +108,6 @@ defmodule Plausible.Billing.PaddleApi do Enum.sort(response, fn %{"payout_date" => d1}, %{"payout_date" => d2} -> Date.compare(Date.from_iso8601!(d1), Date.from_iso8601!(d2)) == :gt end) - |> Enum.take(12) |> then(&{:ok, &1}) else error -> @@ -120,8 +119,10 @@ defmodule Plausible.Billing.PaddleApi do end end - def fetch_prices([_ | _] = product_ids) do - case HTTPClient.impl().get(prices_url(), @headers, %{product_ids: Enum.join(product_ids, ",")}) do + def fetch_prices([_ | _] = product_ids, customer_ip) do + params = %{product_ids: Enum.join(product_ids, ","), customer_ip: customer_ip} + + case HTTPClient.impl().get(prices_url(), @headers, params) do {:ok, %{body: %{"success" => true, "response" => %{"products" => products}}}} -> products = Enum.into(products, %{}, fn %{ diff --git a/lib/plausible/billing/plans.ex b/lib/plausible/billing/plans.ex index fadd095279d6..955bd9947e9e 100644 --- a/lib/plausible/billing/plans.ex +++ b/lib/plausible/billing/plans.ex @@ -45,8 +45,8 @@ defmodule Plausible.Billing.Plans do is_nil(owned_plan) -> @plans_v4 user.subscription && Subscriptions.expired?(user.subscription) -> @plans_v4 owned_plan.kind == :business -> @plans_v4 - owned_plan.generation == 1 -> @plans_v1 - owned_plan.generation == 2 -> @plans_v2 + owned_plan.generation == 1 -> @plans_v1 |> drop_high_plans(owned_plan) + owned_plan.generation == 2 -> @plans_v2 |> drop_high_plans(owned_plan) owned_plan.generation == 3 -> @plans_v3 owned_plan.generation == 4 -> @plans_v4 end @@ -71,7 +71,8 @@ defmodule Plausible.Billing.Plans do plans = if Keyword.get(opts, :with_prices) do - with_prices(plans) + customer_ip = Keyword.fetch!(opts, :customer_ip) + with_prices(plans, customer_ip) else plans end @@ -79,6 +80,14 @@ defmodule Plausible.Billing.Plans do Enum.group_by(plans, & &1.kind) end + @high_legacy_volumes [20_000_000, 50_000_000] + defp drop_high_plans(plans, %Plan{monthly_pageview_limit: current_volume} = _owned) do + plans + |> Enum.reject(fn %Plan{monthly_pageview_limit: plan_volume} -> + plan_volume in @high_legacy_volumes and current_volume < plan_volume + end) + end + @spec yearly_product_ids() :: [String.t()] @doc """ List yearly plans product IDs. @@ -107,7 +116,7 @@ defmodule Plausible.Billing.Plans do end end - def latest_enterprise_plan_with_price(user) do + def latest_enterprise_plan_with_price(user, customer_ip) do enterprise_plan = Repo.one!( from(e in EnterprisePlan, @@ -117,7 +126,7 @@ defmodule Plausible.Billing.Plans do ) ) - {enterprise_plan, get_price_for(enterprise_plan)} + {enterprise_plan, get_price_for(enterprise_plan, customer_ip)} end def subscription_interval(subscription) do @@ -143,10 +152,10 @@ defmodule Plausible.Billing.Plans do response, fills in the `monthly_cost` and `yearly_cost` fields for each given plan and returns the new list of plans with completed information. """ - def with_prices([_ | _] = plans) do + def with_prices([_ | _] = plans, customer_ip) do product_ids = Enum.flat_map(plans, &[&1.monthly_product_id, &1.yearly_product_id]) - case Plausible.Billing.paddle_api().fetch_prices(product_ids) do + case Plausible.Billing.paddle_api().fetch_prices(product_ids, customer_ip) do {:ok, prices} -> Enum.map(plans, fn plan -> plan @@ -171,8 +180,8 @@ defmodule Plausible.Billing.Plans do end end - def get_price_for(%EnterprisePlan{paddle_plan_id: product_id}) do - case Plausible.Billing.paddle_api().fetch_prices([product_id]) do + def get_price_for(%EnterprisePlan{paddle_plan_id: product_id}, customer_ip) do + case Plausible.Billing.paddle_api().fetch_prices([product_id], customer_ip) do {:ok, prices} -> Map.fetch!(prices, product_id) {:error, :api_error} -> nil end @@ -234,7 +243,7 @@ defmodule Plausible.Billing.Plans do [] end - if Enum.any?(Quota.features_usage(user), &(&1 not in growth_features)) do + if Enum.any?(Quota.Usage.features_usage(user), &(&1 not in growth_features)) do :business else :growth diff --git a/lib/plausible/billing/qouta/limits.ex b/lib/plausible/billing/qouta/limits.ex new file mode 100644 index 000000000000..2dcedc62284d --- /dev/null +++ b/lib/plausible/billing/qouta/limits.ex @@ -0,0 +1,120 @@ +defmodule Plausible.Billing.Quota.Limits do + @moduledoc false + + use Plausible + alias Plausible.Users + alias Plausible.Auth.User + alias Plausible.Billing.{Plan, Plans, Subscription, EnterprisePlan, Feature} + alias Plausible.Billing.Feature.{Goals, Props, StatsAPI} + + @type over_limits_error() :: {:over_plan_limits, [limit()]} + @typep limit() :: :site_limit | :pageview_limit | :team_member_limit + @pageview_allowance_margin 0.1 + + on_ee do + @limit_sites_since ~D[2021-05-05] + @site_limit_for_trials 10 + @team_member_limit_for_trials 3 + + @spec site_limit(User.t()) :: non_neg_integer() | :unlimited + def site_limit(user) do + if Timex.before?(user.inserted_at, @limit_sites_since) do + :unlimited + else + get_site_limit_from_plan(user) + end + end + + defp get_site_limit_from_plan(user) do + user = Users.with_subscription(user) + + case Plans.get_subscription_plan(user.subscription) do + %{site_limit: site_limit} -> site_limit + :free_10k -> 50 + nil -> @site_limit_for_trials + end + end + + @spec team_member_limit(User.t()) :: non_neg_integer() + def team_member_limit(user) do + user = Users.with_subscription(user) + + case Plans.get_subscription_plan(user.subscription) do + %{team_member_limit: limit} -> limit + :free_10k -> :unlimited + nil -> @team_member_limit_for_trials + end + end + else + def site_limit(_) do + :unlimited + end + + def team_member_limit(_) do + :unlimited + end + end + + @monthly_pageview_limit_for_free_10k 10_000 + @monthly_pageview_limit_for_trials :unlimited + + @spec monthly_pageview_limit(User.t() | Subscription.t()) :: + non_neg_integer() | :unlimited + def monthly_pageview_limit(%User{} = user) do + user = Users.with_subscription(user) + monthly_pageview_limit(user.subscription) + end + + def monthly_pageview_limit(subscription) do + case Plans.get_subscription_plan(subscription) do + %EnterprisePlan{monthly_pageview_limit: limit} -> + limit + + %Plan{monthly_pageview_limit: limit} -> + limit + + :free_10k -> + @monthly_pageview_limit_for_free_10k + + _any -> + if subscription do + Sentry.capture_message("Unknown monthly pageview limit for plan", + extra: %{paddle_plan_id: subscription.paddle_plan_id} + ) + end + + @monthly_pageview_limit_for_trials + end + end + + def pageview_limit_with_margin(limit, margin \\ nil) do + margin = if margin, do: margin, else: @pageview_allowance_margin + ceil(limit * (1 + margin)) + end + + @doc """ + Returns a list of features the user can use. Trial users have the + ability to use all features during their trial. + """ + def allowed_features_for(user) do + user = Users.with_subscription(user) + + case Plans.get_subscription_plan(user.subscription) do + %EnterprisePlan{features: features} -> + features + + %Plan{features: features} -> + features + + :free_10k -> + [Goals, Props, StatsAPI] + + nil -> + if Users.on_trial?(user) do + Feature.list() + else + [Goals] + end + end + end +end diff --git a/lib/plausible/billing/qouta/quota.ex b/lib/plausible/billing/qouta/quota.ex new file mode 100644 index 000000000000..cf129fbe8f47 --- /dev/null +++ b/lib/plausible/billing/qouta/quota.ex @@ -0,0 +1,125 @@ +defmodule Plausible.Billing.Quota do + @moduledoc """ + This module provides functions to work with plans usage and limits. + """ + + use Plausible + alias Plausible.Users + alias Plausible.Auth.User + alias Plausible.Billing.{Plan, Plans, EnterprisePlan} + alias Plausible.Billing.Quota.{Usage, Limits} + + @doc """ + Enterprise plans are always allowed to add more sites (even when + over limit) to avoid service disruption. Their usage is checked + in a background job instead (see `check_usage.ex`). + """ + def ensure_can_add_new_site(user) do + user = Users.with_subscription(user) + + case Plans.get_subscription_plan(user.subscription) do + %EnterprisePlan{} -> + :ok + + _ -> + usage = Usage.site_usage(user) + limit = Limits.site_limit(user) + + if below_limit?(usage, limit), do: :ok, else: {:error, {:over_limit, limit}} + end + end + + @doc """ + Ensures that the given user (or the usage map) is within the limits + of the given plan. + + An `opts` argument can be passed with `ignore_pageview_limit: true` + which bypasses the pageview limit check and returns `:ok` as long as + the other limits are not exceeded. + """ + @spec ensure_within_plan_limits(User.t() | map(), struct() | atom() | nil, Keyword.t()) :: + :ok | {:error, Limits.over_limits_error()} + def ensure_within_plan_limits(user_or_usage, plan, opts \\ []) + + def ensure_within_plan_limits(%User{} = user, %plan_mod{} = plan, opts) + when plan_mod in [Plan, EnterprisePlan] do + ensure_within_plan_limits(Usage.usage(user), plan, opts) + end + + def ensure_within_plan_limits(usage, %plan_mod{} = plan, opts) + when plan_mod in [Plan, EnterprisePlan] do + case exceeded_limits(usage, plan, opts) do + [] -> :ok + exceeded_limits -> {:error, {:over_plan_limits, exceeded_limits}} + end + end + + def ensure_within_plan_limits(_, _, _), do: :ok + + def eligible_for_upgrade?(usage), do: usage.sites > 0 + + defp exceeded_limits(usage, plan, opts) do + for {limit, exceeded?} <- [ + {:team_member_limit, not within_limit?(usage.team_members, plan.team_member_limit)}, + {:site_limit, not within_limit?(usage.sites, plan.site_limit)}, + {:monthly_pageview_limit, + exceeds_monthly_pageview_limit?(usage.monthly_pageviews, plan, opts)} + ], + exceeded? do + limit + end + end + + defp exceeds_monthly_pageview_limit?(usage, plan, opts) do + if Keyword.get(opts, :ignore_pageview_limit) do + false + else + case usage do + %{last_30_days: %{total: total}} -> + margin = Keyword.get(opts, :pageview_allowance_margin) + limit = Limits.pageview_limit_with_margin(plan.monthly_pageview_limit, margin) + !within_limit?(total, limit) + + cycles_usage -> + exceeds_last_two_usage_cycles?(cycles_usage, plan.monthly_pageview_limit) + end + end + end + + @spec exceeds_last_two_usage_cycles?(Usage.cycles_usage(), non_neg_integer()) :: boolean() + def exceeds_last_two_usage_cycles?(cycles_usage, allowed_volume) do + exceeded = exceeded_cycles(cycles_usage, allowed_volume) + :penultimate_cycle in exceeded && :last_cycle in exceeded + end + + @spec exceeded_cycles(Usage.cycles_usage(), non_neg_integer()) :: list() + def exceeded_cycles(cycles_usage, allowed_volume) do + limit = Limits.pageview_limit_with_margin(allowed_volume) + + Enum.reduce(cycles_usage, [], fn {cycle, %{total: total}}, exceeded_cycles -> + if below_limit?(total, limit) do + exceeded_cycles + else + exceeded_cycles ++ [cycle] + end + end) + end + + @spec below_limit?(non_neg_integer(), non_neg_integer() | :unlimited) :: boolean() + @doc """ + Returns whether the usage is below the limit or not. + Returns false if usage is equal to the limit. + """ + def below_limit?(usage, limit) do + if limit == :unlimited, do: true, else: usage < limit + end + + @spec within_limit?(non_neg_integer(), non_neg_integer() | :unlimited) :: boolean() + @doc """ + Returns whether the usage is within the limit or not. + Returns true if usage is equal to the limit. + """ + def within_limit?(usage, limit) do + if limit == :unlimited, do: true, else: usage <= limit + end +end diff --git a/lib/plausible/billing/qouta/usage.ex b/lib/plausible/billing/qouta/usage.ex new file mode 100644 index 000000000000..4346f86a0c07 --- /dev/null +++ b/lib/plausible/billing/qouta/usage.ex @@ -0,0 +1,294 @@ +defmodule Plausible.Billing.Quota.Usage do + @moduledoc false + + use Plausible + import Ecto.Query + alias Plausible.Users + alias Plausible.Auth.User + alias Plausible.Site + alias Plausible.Billing.{Subscriptions, Feature} + + @type cycles_usage() :: %{cycle() => usage_cycle()} + + @typep cycle :: :current_cycle | :last_cycle | :penultimate_cycle + @typep last_30_days_usage() :: %{:last_30_days => usage_cycle()} + @typep monthly_pageview_usage() :: cycles_usage() | last_30_days_usage() + + @typep usage_cycle :: %{ + date_range: Date.Range.t(), + pageviews: non_neg_integer(), + custom_events: non_neg_integer(), + total: non_neg_integer() + } + + @doc """ + Returns a full usage report for the user. + + ### Options + + * `pending_ownership_site_ids` - a list of site IDs from which to count + additional usage. This allows us to look at the total usage from pending + ownerships and owned sites at the same time, which is useful, for example, + when deciding whether to let the user upgrade to a plan, or accept a site + ownership. + + * `with_features` - when `true`, the returned map will contain features + usage. Also counts usage from `pending_ownership_site_ids` if that option + is given. + """ + def usage(user, opts \\ []) do + owned_site_ids = Plausible.Sites.owned_site_ids(user) + pending_ownership_site_ids = Keyword.get(opts, :pending_ownership_site_ids, []) + all_site_ids = Enum.uniq(owned_site_ids ++ pending_ownership_site_ids) + + basic_usage = %{ + monthly_pageviews: monthly_pageview_usage(user, all_site_ids), + team_members: + team_member_usage(user, pending_ownership_site_ids: pending_ownership_site_ids), + sites: length(all_site_ids) + } + + if Keyword.get(opts, :with_features) == true do + basic_usage + |> Map.put(:features, features_usage(user, all_site_ids)) + else + basic_usage + end + end + + @spec site_usage(User.t()) :: non_neg_integer() + @doc """ + Returns the number of sites the given user owns. + """ + def site_usage(user) do + Plausible.Sites.owned_sites_count(user) + end + + @doc """ + Queries the ClickHouse database for the monthly pageview usage. If the given user's + subscription is `active`, `past_due`, or a `deleted` (but not yet expired), a map + with the following structure is returned: + + ```elixir + %{ + current_cycle: usage_cycle(), + last_cycle: usage_cycle(), + penultimate_cycle: usage_cycle() + } + ``` + + In all other cases of the subscription status (or a `free_10k` subscription which + does not have a `last_bill_date` defined) - the following structure is returned: + + ```elixir + %{last_30_days: usage_cycle()} + ``` + + Given only a user as input, the usage is queried from across all the sites that the + user owns. Alternatively, given an optional argument of `site_ids`, the usage from + across all those sites is queried instead. + """ + @spec monthly_pageview_usage(User.t(), list() | nil) :: monthly_pageview_usage() + def monthly_pageview_usage(user, site_ids \\ nil) + + def monthly_pageview_usage(user, nil) do + monthly_pageview_usage(user, Plausible.Sites.owned_site_ids(user)) + end + + def monthly_pageview_usage(user, site_ids) do + active_subscription? = Subscriptions.active?(user.subscription) + + if active_subscription? && user.subscription.last_bill_date do + [:current_cycle, :last_cycle, :penultimate_cycle] + |> Task.async_stream(fn cycle -> + %{cycle => usage_cycle(user, cycle, site_ids)} + end) + |> Enum.map(fn {:ok, cycle_usage} -> cycle_usage end) + |> Enum.reduce(%{}, &Map.merge/2) + else + %{last_30_days: usage_cycle(user, :last_30_days, site_ids)} + end + end + + @spec usage_cycle(User.t(), :last_30_days | cycle(), list() | nil, Date.t()) :: usage_cycle() + def usage_cycle(user, cycle, owned_site_ids \\ nil, today \\ Timex.today()) + + def usage_cycle(user, cycle, nil, today) do + usage_cycle(user, cycle, Plausible.Sites.owned_site_ids(user), today) + end + + def usage_cycle(_user, :last_30_days, owned_site_ids, today) do + date_range = Date.range(Timex.shift(today, days: -30), today) + + {pageviews, custom_events} = + Plausible.Stats.Clickhouse.usage_breakdown(owned_site_ids, date_range) + + %{ + date_range: date_range, + pageviews: pageviews, + custom_events: custom_events, + total: pageviews + custom_events + } + end + + def usage_cycle(user, cycle, owned_site_ids, today) do + user = Users.with_subscription(user) + last_bill_date = user.subscription.last_bill_date + + normalized_last_bill_date = + Timex.shift(last_bill_date, months: Timex.diff(today, last_bill_date, :months)) + + date_range = + case cycle do + :current_cycle -> + Date.range( + normalized_last_bill_date, + Timex.shift(normalized_last_bill_date, months: 1, days: -1) + ) + + :last_cycle -> + Date.range( + Timex.shift(normalized_last_bill_date, months: -1), + Timex.shift(normalized_last_bill_date, days: -1) + ) + + :penultimate_cycle -> + Date.range( + Timex.shift(normalized_last_bill_date, months: -2), + Timex.shift(normalized_last_bill_date, days: -1, months: -1) + ) + end + + {pageviews, custom_events} = + Plausible.Stats.Clickhouse.usage_breakdown(owned_site_ids, date_range) + + %{ + date_range: date_range, + pageviews: pageviews, + custom_events: custom_events, + total: pageviews + custom_events + } + end + + @spec team_member_usage(User.t(), Keyword.t()) :: non_neg_integer() + @doc """ + Returns the total count of team members associated with the user's sites. + + * The given user (i.e. the owner) is not counted as a team member. + + * Pending invitations (but not ownership transfers) are counted as team + members even before accepted. + + * Users are counted uniquely - i.e. even if an account is associated with + many sites owned by the given user, they still count as one team member. + + ### Options + + * `exclude_emails` - a list of emails to not count towards the usage. This + allows us to exclude a user from being counted as a team member when + checking whether a site invitation can be created for that same user. + + * `pending_ownership_site_ids` - a list of site IDs from which to count + additional team member usage. Without this option, usage is queried only + across sites owned by the given user. + """ + def team_member_usage(user, opts \\ []) + + def team_member_usage(%User{} = user, opts) do + exclude_emails = Keyword.get(opts, :exclude_emails, []) ++ [user.email] + + q = + user + |> Plausible.Sites.owned_site_ids() + |> query_team_member_emails() + + q = + case Keyword.get(opts, :pending_ownership_site_ids) do + [_ | _] = site_ids -> union(q, ^query_team_member_emails(site_ids)) + _ -> q + end + + from(u in subquery(q), + where: u.email not in ^exclude_emails, + distinct: u.email + ) + |> Plausible.Repo.aggregate(:count) + end + + def query_team_member_emails(site_ids) do + memberships_q = + from sm in Site.Membership, + where: sm.site_id in ^site_ids, + inner_join: u in assoc(sm, :user), + select: %{email: u.email} + + invitations_q = + from i in Plausible.Auth.Invitation, + where: i.site_id in ^site_ids and i.role != :owner, + select: %{email: i.email} + + union(memberships_q, ^invitations_q) + end + + @spec features_usage(User.t() | nil, list() | nil) :: [atom()] + @doc """ + Given only a user, this function returns the features used across all the + sites this user owns + StatsAPI if the user has a configured Stats API key. + + Given a user, and a list of site_ids, returns the features used by those + sites instead + StatsAPI if the user has a configured Stats API key. + + The user can also be passed as `nil`, in which case we will never return + Stats API as a used feature. + """ + def features_usage(user, site_ids \\ nil) + + def features_usage(%User{} = user, nil) do + site_ids = Plausible.Sites.owned_site_ids(user) + features_usage(user, site_ids) + end + + def features_usage(%User{} = user, site_ids) when is_list(site_ids) do + site_scoped_feature_usage = features_usage(nil, site_ids) + + stats_api_used? = + from(a in Plausible.Auth.ApiKey, where: a.user_id == ^user.id) + |> Plausible.Repo.exists?() + + if stats_api_used? do + site_scoped_feature_usage ++ [Feature.StatsAPI] + else + site_scoped_feature_usage + end + end + + def features_usage(nil, site_ids) when is_list(site_ids) do + props_usage_q = + from s in Site, + where: s.id in ^site_ids and fragment("cardinality(?) > 0", s.allowed_event_props) + + revenue_goals_usage_q = + from g in Plausible.Goal, + where: g.site_id in ^site_ids and not is_nil(g.currency) + + queries = + on_ee do + funnels_usage_q = from f in "funnels", where: f.site_id in ^site_ids + + [ + {Feature.Props, props_usage_q}, + {Feature.Funnels, funnels_usage_q}, + {Feature.RevenueGoals, revenue_goals_usage_q} + ] + else + [ + {Feature.Props, props_usage_q}, + {Feature.RevenueGoals, revenue_goals_usage_q} + ] + end + + Enum.reduce(queries, [], fn {feature, query}, acc -> + if Plausible.Repo.exists?(query), do: acc ++ [feature], else: acc + end) + end +end diff --git a/lib/plausible/billing/quota.ex b/lib/plausible/billing/quota.ex deleted file mode 100644 index a6505e5df27c..000000000000 --- a/lib/plausible/billing/quota.ex +++ /dev/null @@ -1,511 +0,0 @@ -defmodule Plausible.Billing.Quota do - @moduledoc """ - This module provides functions to work with plans usage and limits. - """ - - use Plausible - import Ecto.Query - alias Plausible.Users - alias Plausible.Auth.User - alias Plausible.Site - alias Plausible.Billing.{Plan, Plans, Subscription, Subscriptions, EnterprisePlan, Feature} - alias Plausible.Billing.Feature.{Goals, RevenueGoals, Funnels, Props, StatsAPI} - - @type limit() :: :site_limit | :pageview_limit | :team_member_limit - - @type over_limits_error() :: {:over_plan_limits, [limit()]} - - @type monthly_pageview_usage() :: %{period() => usage_cycle()} - - @type period :: :last_30_days | :current_cycle | :last_cycle | :penultimate_cycle - - @type usage_cycle :: %{ - date_range: Date.Range.t(), - pageviews: non_neg_integer(), - custom_events: non_neg_integer(), - total: non_neg_integer() - } - - @pageview_allowance_margin 0.1 - - def pageview_allowance_margin(), do: @pageview_allowance_margin - - def usage(user, opts \\ []) do - basic_usage = %{ - monthly_pageviews: monthly_pageview_usage(user), - team_members: team_member_usage(user), - sites: site_usage(user) - } - - if Keyword.get(opts, :with_features) == true do - basic_usage - |> Map.put(:features, features_usage(user)) - else - basic_usage - end - end - - on_ee do - @limit_sites_since ~D[2021-05-05] - @site_limit_for_trials 10 - @team_member_limit_for_trials 3 - - @spec site_limit(User.t()) :: non_neg_integer() | :unlimited - def site_limit(user) do - if Timex.before?(user.inserted_at, @limit_sites_since) do - :unlimited - else - get_site_limit_from_plan(user) - end - end - - defp get_site_limit_from_plan(user) do - user = Users.with_subscription(user) - - case Plans.get_subscription_plan(user.subscription) do - %{site_limit: site_limit} -> site_limit - :free_10k -> 50 - nil -> @site_limit_for_trials - end - end - - @spec team_member_limit(User.t()) :: non_neg_integer() - def team_member_limit(user) do - user = Users.with_subscription(user) - - case Plans.get_subscription_plan(user.subscription) do - %{team_member_limit: limit} -> limit - :free_10k -> :unlimited - nil -> @team_member_limit_for_trials - end - end - else - def site_limit(_) do - :unlimited - end - - def team_member_limit(_) do - :unlimited - end - end - - @spec site_usage(User.t()) :: non_neg_integer() - @doc """ - Returns the number of sites the given user owns. - """ - def site_usage(user) do - Plausible.Sites.owned_sites_count(user) - end - - @doc """ - Enterprise plans are always allowed to add more sites (even when - over limit) to avoid service disruption. Their usage is checked - in a background job instead (see `check_usage.ex`). - """ - def ensure_can_add_new_site(user) do - user = Users.with_subscription(user) - - case Plans.get_subscription_plan(user.subscription) do - %EnterprisePlan{} -> - :ok - - _ -> - usage = site_usage(user) - limit = site_limit(user) - - if below_limit?(usage, limit), do: :ok, else: {:error, {:over_limit, limit}} - end - end - - @monthly_pageview_limit_for_free_10k 10_000 - @monthly_pageview_limit_for_trials :unlimited - - @spec monthly_pageview_limit(User.t() | Subscription.t()) :: - non_neg_integer() | :unlimited - def monthly_pageview_limit(%User{} = user) do - user = Users.with_subscription(user) - monthly_pageview_limit(user.subscription) - end - - def monthly_pageview_limit(subscription) do - case Plans.get_subscription_plan(subscription) do - %EnterprisePlan{monthly_pageview_limit: limit} -> - limit - - %Plan{monthly_pageview_limit: limit} -> - limit - - :free_10k -> - @monthly_pageview_limit_for_free_10k - - _any -> - if subscription do - Sentry.capture_message("Unknown monthly pageview limit for plan", - extra: %{paddle_plan_id: subscription.paddle_plan_id} - ) - end - - @monthly_pageview_limit_for_trials - end - end - - @doc """ - Queries the ClickHouse database for the monthly pageview usage. If the given user's - subscription is `active`, `past_due`, or a `deleted` (but not yet expired), a map - with the following structure is returned: - - ```elixir - %{ - current_cycle: usage_cycle(), - last_cycle: usage_cycle(), - penultimate_cycle: usage_cycle() - } - ``` - - In all other cases of the subscription status (or a `free_10k` subscription which - does not have a `last_bill_date` defined) - the following structure is returned: - - ```elixir - %{last_30_days: usage_cycle()} - ``` - - Given only a user as input, the usage is queried from across all the sites that the - user owns. Alternatively, given an optional argument of `site_ids`, the usage from - across all those sites is queried instead. - """ - @spec monthly_pageview_usage(User.t(), list() | nil) :: monthly_pageview_usage() - def monthly_pageview_usage(user, site_ids \\ nil) - - def monthly_pageview_usage(user, nil) do - monthly_pageview_usage(user, Plausible.Sites.owned_site_ids(user)) - end - - def monthly_pageview_usage(user, site_ids) do - active_subscription? = Subscriptions.active?(user.subscription) - - if active_subscription? && user.subscription.last_bill_date do - [:current_cycle, :last_cycle, :penultimate_cycle] - |> Task.async_stream(fn cycle -> - %{cycle => usage_cycle(user, cycle, site_ids)} - end) - |> Enum.map(fn {:ok, cycle_usage} -> cycle_usage end) - |> Enum.reduce(%{}, &Map.merge/2) - else - %{last_30_days: usage_cycle(user, :last_30_days, site_ids)} - end - end - - @spec usage_cycle(User.t(), period(), list() | nil, Date.t()) :: usage_cycle() - - def usage_cycle(user, cycle, owned_site_ids \\ nil, today \\ Timex.today()) - - def usage_cycle(user, cycle, nil, today) do - usage_cycle(user, cycle, Plausible.Sites.owned_site_ids(user), today) - end - - def usage_cycle(_user, :last_30_days, owned_site_ids, today) do - date_range = Date.range(Timex.shift(today, days: -30), today) - - {pageviews, custom_events} = - Plausible.Stats.Clickhouse.usage_breakdown(owned_site_ids, date_range) - - %{ - date_range: date_range, - pageviews: pageviews, - custom_events: custom_events, - total: pageviews + custom_events - } - end - - def usage_cycle(user, cycle, owned_site_ids, today) do - user = Users.with_subscription(user) - last_bill_date = user.subscription.last_bill_date - - normalized_last_bill_date = - Timex.shift(last_bill_date, months: Timex.diff(today, last_bill_date, :months)) - - date_range = - case cycle do - :current_cycle -> - Date.range( - normalized_last_bill_date, - Timex.shift(normalized_last_bill_date, months: 1, days: -1) - ) - - :last_cycle -> - Date.range( - Timex.shift(normalized_last_bill_date, months: -1), - Timex.shift(normalized_last_bill_date, days: -1) - ) - - :penultimate_cycle -> - Date.range( - Timex.shift(normalized_last_bill_date, months: -2), - Timex.shift(normalized_last_bill_date, days: -1, months: -1) - ) - end - - {pageviews, custom_events} = - Plausible.Stats.Clickhouse.usage_breakdown(owned_site_ids, date_range) - - %{ - date_range: date_range, - pageviews: pageviews, - custom_events: custom_events, - total: pageviews + custom_events - } - end - - @spec team_member_usage(User.t()) :: integer() - @doc """ - Returns the total count of team members associated with the user's sites. - - * The given user (i.e. the owner) is not counted as a team member. - - * Pending invitations are counted as team members even before accepted. - - * Users are counted uniquely - i.e. even if an account is associated with - many sites owned by the given user, they still count as one team member. - - * Specific e-mails can be excluded from the count, so that where necessary, - we can ensure inviting the same person(s) to more than 1 sites is allowed - """ - def team_member_usage(user, opts \\ []) do - {:ok, opts} = Keyword.validate(opts, site: nil, exclude_emails: []) - - user - |> team_member_usage_query(opts) - |> Plausible.Repo.aggregate(:count) - end - - defp team_member_usage_query(user, opts) do - owned_sites_query = owned_sites_query(user) - - excluded_emails = - opts - |> Keyword.get(:exclude_emails, []) - |> List.wrap() - - site = opts[:site] - - owned_sites_query = - if site do - where(owned_sites_query, [os], os.site_id == ^site.id) - else - owned_sites_query - end - - team_members_query = - from os in subquery(owned_sites_query), - inner_join: sm in Site.Membership, - on: sm.site_id == os.site_id, - inner_join: u in assoc(sm, :user), - where: sm.role != :owner, - select: u.email - - team_members_query = - if excluded_emails != [] do - team_members_query |> where([..., u], u.email not in ^excluded_emails) - else - team_members_query - end - - query = - from i in Plausible.Auth.Invitation, - inner_join: os in subquery(owned_sites_query), - on: i.site_id == os.site_id, - where: i.role != :owner, - select: i.email, - union: ^team_members_query - - if excluded_emails != [] do - query - |> where([i], i.email not in ^excluded_emails) - else - query - end - end - - @spec features_usage(User.t() | Site.t()) :: [atom()] - @doc """ - Given a user, this function returns the features used across all the sites - this user owns + StatsAPI if the user has a configured Stats API key. - - Given a site, returns the features used by the site. - """ - def features_usage(%User{} = user) do - props_usage_query = - from s in Site, - inner_join: os in subquery(owned_sites_query(user)), - on: s.id == os.site_id, - where: fragment("cardinality(?) > 0", s.allowed_event_props) - - revenue_goals_usage = - from g in Plausible.Goal, - inner_join: os in subquery(owned_sites_query(user)), - on: g.site_id == os.site_id, - where: not is_nil(g.currency) - - stats_api_usage = from a in Plausible.Auth.ApiKey, where: a.user_id == ^user.id - - queries = - on_ee do - funnels_usage_query = - from f in "funnels", - inner_join: os in subquery(owned_sites_query(user)), - on: f.site_id == os.site_id - - [ - {Props, props_usage_query}, - {Funnels, funnels_usage_query}, - {RevenueGoals, revenue_goals_usage}, - {StatsAPI, stats_api_usage} - ] - else - [ - {Props, props_usage_query}, - {RevenueGoals, revenue_goals_usage}, - {StatsAPI, stats_api_usage} - ] - end - - Enum.reduce(queries, [], fn {feature, query}, acc -> - if Plausible.Repo.exists?(query), do: acc ++ [feature], else: acc - end) - end - - def features_usage(%Site{} = site) do - props_exist = is_list(site.allowed_event_props) && site.allowed_event_props != [] - - funnels_exist = - on_ee do - Plausible.Repo.exists?(from f in Plausible.Funnel, where: f.site_id == ^site.id) - else - false - end - - revenue_goals_exist = - Plausible.Repo.exists?( - from g in Plausible.Goal, where: g.site_id == ^site.id and not is_nil(g.currency) - ) - - used_features = [ - {Props, props_exist}, - {Funnels, funnels_exist}, - {RevenueGoals, revenue_goals_exist} - ] - - for {f_mod, used?} <- used_features, used?, f_mod.enabled?(site), do: f_mod - end - - @doc """ - Ensures that the given user (or the usage map) is within the limits - of the given plan. - - An `opts` argument can be passed with `ignore_pageview_limit: true` - which bypasses the pageview limit check and returns `:ok` as long as - the other limits are not exceeded. - """ - @spec ensure_within_plan_limits(User.t() | map(), struct() | atom() | nil, Keyword.t()) :: - :ok | {:error, over_limits_error()} - def ensure_within_plan_limits(user_or_usage, plan, opts \\ []) - - def ensure_within_plan_limits(%User{} = user, %plan_mod{} = plan, opts) - when plan_mod in [Plan, EnterprisePlan] do - ensure_within_plan_limits(usage(user), plan, opts) - end - - def ensure_within_plan_limits(usage, %plan_mod{} = plan, opts) - when plan_mod in [Plan, EnterprisePlan] do - case exceeded_limits(usage, plan, opts) do - [] -> :ok - exceeded_limits -> {:error, {:over_plan_limits, exceeded_limits}} - end - end - - def ensure_within_plan_limits(_, _, _), do: :ok - - defp exceeded_limits(usage, plan, opts) do - for {limit, exceeded?} <- [ - {:team_member_limit, not within_limit?(usage.team_members, plan.team_member_limit)}, - {:site_limit, not within_limit?(usage.sites, plan.site_limit)}, - {:monthly_pageview_limit, - exceeds_monthly_pageview_limit?(usage.monthly_pageviews, plan, opts)} - ], - exceeded? do - limit - end - end - - defp exceeds_monthly_pageview_limit?(usage, plan, opts) do - if Keyword.get(opts, :ignore_pageview_limit) do - false - else - case usage do - %{last_30_days: %{total: total}} -> - !within_limit?(total, pageview_limit_with_margin(plan, opts)) - - billing_cycles_usage -> - Plausible.Workers.CheckUsage.exceeds_last_two_usage_cycles?( - billing_cycles_usage, - plan.monthly_pageview_limit - ) - end - end - end - - defp pageview_limit_with_margin(%{monthly_pageview_limit: limit}, opts) do - margin = Keyword.get(opts, :pageview_allowance_margin, @pageview_allowance_margin) - ceil(limit * (1 + margin)) - end - - @doc """ - Returns a list of features the user can use. Trial users have the - ability to use all features during their trial. - """ - def allowed_features_for(user) do - user = Users.with_subscription(user) - - case Plans.get_subscription_plan(user.subscription) do - %EnterprisePlan{features: features} -> - features - - %Plan{features: features} -> - features - - :free_10k -> - [Goals, Props, StatsAPI] - - nil -> - if Users.on_trial?(user) do - Feature.list() - else - [Goals] - end - end - end - - defp owned_sites_query(user) do - from sm in Site.Membership, - where: sm.role == :owner and sm.user_id == ^user.id, - select: %{site_id: sm.site_id} - end - - @spec below_limit?(non_neg_integer(), non_neg_integer() | :unlimited) :: boolean() - @doc """ - Returns whether the usage is below the limit or not. - Returns false if usage is equal to the limit. - """ - def below_limit?(usage, limit) do - if limit == :unlimited, do: true, else: usage < limit - end - - @spec within_limit?(non_neg_integer(), non_neg_integer() | :unlimited) :: boolean() - @doc """ - Returns whether the usage is within the limit or not. - Returns true if usage is equal to the limit. - """ - def within_limit?(usage, limit) do - if limit == :unlimited, do: true, else: usage <= limit - end -end diff --git a/lib/plausible/billing/site_locker.ex b/lib/plausible/billing/site_locker.ex index 67ba076b1780..11b2c4228009 100644 --- a/lib/plausible/billing/site_locker.ex +++ b/lib/plausible/billing/site_locker.ex @@ -68,7 +68,7 @@ defmodule Plausible.Billing.SiteLocker do @spec send_grace_period_end_email(Plausible.Auth.User.t()) :: Plausible.Mailer.result() def send_grace_period_end_email(user) do - usage = Plausible.Billing.Quota.monthly_pageview_usage(user) + usage = Plausible.Billing.Quota.Usage.monthly_pageview_usage(user) suggested_plan = Plausible.Billing.Plans.suggest(user, usage.last_cycle.total) PlausibleWeb.Email.dashboard_locked(user, usage, suggested_plan) diff --git a/lib/plausible/cache/adapter.ex b/lib/plausible/cache/adapter.ex index 100965165b6f..65f8c254c4d0 100644 --- a/lib/plausible/cache/adapter.ex +++ b/lib/plausible/cache/adapter.ex @@ -52,6 +52,15 @@ defmodule Plausible.Cache.Adapter do nil end + @spec fetch(atom(), any(), (-> any())) :: any() + def fetch(cache_name, key, fallback_fn) do + ConCache.fetch_or_store(cache_name, key, fallback_fn) + catch + :exit, _ -> + Logger.error("Error fetching key from '#{inspect(cache_name)}'") + nil + end + @spec put(atom(), any(), any()) :: any() def put(cache_name, key, value) do :ok = ConCache.put(cache_name, key, value) diff --git a/lib/plausible/crm_extensions.ex b/lib/plausible/crm_extensions.ex new file mode 100644 index 000000000000..b7feca526dfe --- /dev/null +++ b/lib/plausible/crm_extensions.ex @@ -0,0 +1,128 @@ +defmodule Plausible.CrmExtensions do + @moduledoc """ + Extensions for Kaffy CRM + """ + + use Plausible + + on_ee do + def javascripts(%{assigns: %{context: "auth", resource: "user", entry: %{} = user}}) do + [ + Phoenix.HTML.raw(""" + + """) + ] + end + + def javascripts(%{assigns: %{context: "sites", resource: "site", entry: %{domain: domain}}}) do + base_url = PlausibleWeb.Endpoint.url() + + [ + Phoenix.HTML.raw(""" + + """) + ] + end + + def javascripts(%{assigns: %{context: "billing", resource: "enterprise_plan", changeset: %{}}}) do + [ + Phoenix.HTML.raw(""" + + """), + Phoenix.HTML.raw(""" + + """) + ] + end + end + + def javascripts(_) do + [] + end +end diff --git a/lib/plausible/data_migration/clean_up_demo_site_referrer_source.ex b/lib/plausible/data_migration/clean_up_demo_site_referrer_source.ex new file mode 100644 index 000000000000..50844bc2644f --- /dev/null +++ b/lib/plausible/data_migration/clean_up_demo_site_referrer_source.ex @@ -0,0 +1,25 @@ +defmodule Plausible.DataMigration.CleanUpDemoSiteReferrerSource do + @moduledoc """ + Clean up referrer_source entries for demo site with + `Direct / None` for value populated by dogfooding + Plausible stats. + """ + + alias Plausible.IngestRepo + alias Plausible.Repo + + def run(timeout \\ 60_000) do + demo_domain = PlausibleWeb.Endpoint.host() + %{id: demo_site_id} = Repo.get_by(Plausible.Site, domain: demo_domain) + + for table <- ["sessions_v2", "events_v2"] do + IngestRepo.query!( + "ALTER TABLE {$0:Identifier} UPDATE referrer_source = '' WHERE " <> + "site_id = {$1:UInt64} AND referrer_source = 'Direct / None'", + [table, demo_site_id], + settings: [mutations_sync: 1], + timeout: timeout + ) + end + end +end diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex index 484ed843909b..ee9bc2157866 100644 --- a/lib/plausible/google/search_console/filters.ex +++ b/lib/plausible/google/search_console/filters.ex @@ -23,23 +23,14 @@ defmodule Plausible.Google.SearchConsole.Filters do transform_filter(property, [op, "visit:entry_page" | rest]) end - defp transform_filter(property, [:is, "visit:entry_page", page]) when is_binary(page) do - %{dimension: "page", expression: property_url(property, page)} - end - - defp transform_filter(property, [:member, "visit:entry_page", pages]) when is_list(pages) do + defp transform_filter(property, [:is, "visit:entry_page", pages]) when is_list(pages) do expression = Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end) %{dimension: "page", operator: "includingRegex", expression: expression} end - defp transform_filter(property, [:matches, "visit:entry_page", page]) when is_binary(page) do - page = page_regex(property_url(property, page)) - %{dimension: "page", operator: "includingRegex", expression: page} - end - - defp transform_filter(property, [:matches_member, "visit:entry_page", pages]) + defp transform_filter(property, [:matches, "visit:entry_page", pages]) when is_list(pages) do expression = Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end) @@ -47,20 +38,12 @@ defmodule Plausible.Google.SearchConsole.Filters do %{dimension: "page", operator: "includingRegex", expression: expression} end - defp transform_filter(_property, [:is, "visit:screen", device]) when is_binary(device) do - %{dimension: "device", expression: search_console_device(device)} - end - - defp transform_filter(_property, [:member, "visit:screen", devices]) when is_list(devices) do - expression = devices |> Enum.join("|") + defp transform_filter(_property, [:is, "visit:screen", devices]) when is_list(devices) do + expression = Enum.map_join(devices, "|", &search_console_device/1) %{dimension: "device", operator: "includingRegex", expression: expression} end - defp transform_filter(_property, [:is, "visit:country", country]) when is_binary(country) do - %{dimension: "country", expression: search_console_country(country)} - end - - defp transform_filter(_property, [:member, "visit:country", countries]) + defp transform_filter(_property, [:is, "visit:country", countries]) when is_list(countries) do expression = Enum.map_join(countries, "|", &search_console_country/1) %{dimension: "country", operator: "includingRegex", expression: expression} diff --git a/lib/plausible/google/ua/api.ex b/lib/plausible/google/ua/api.ex index e93f55fafdaa..9473865ca6f8 100644 --- a/lib/plausible/google/ua/api.ex +++ b/lib/plausible/google/ua/api.ex @@ -14,7 +14,7 @@ defmodule Plausible.Google.UA.API do expires_at :: String.t() } - @per_page 7_500 + @per_page 100_000 @backoff_factor :timer.seconds(10) @max_attempts 5 diff --git a/lib/plausible/open_telemetry/sampler.ex b/lib/plausible/open_telemetry/sampler.ex index e6a4599049d1..a56c0503a6a0 100644 --- a/lib/plausible/open_telemetry/sampler.ex +++ b/lib/plausible/open_telemetry/sampler.ex @@ -18,7 +18,7 @@ defmodule Plausible.OpenTelemetry.Sampler do require OpenTelemetry.Tracer, as: Tracer @routes_to_ignore ["/api/event", "/api/event/"] - @tables_to_ignore ["oban_jobs"] + @tables_to_ignore ["oban_jobs", "site_imports"] @impl true def setup(%{ratio: ratio}) when is_number(ratio) do diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex index e6c593e8454b..fe3b2b1b0600 100644 --- a/lib/plausible/site/admin.ex +++ b/lib/plausible/site/admin.ex @@ -17,8 +17,7 @@ defmodule Plausible.SiteAdmin do def custom_index_query(_conn, _schema, query) do from(r in query, inner_join: o in assoc(r, :owner), - inner_join: m in assoc(r, :memberships), - preload: [owner: o, memberships: {m, :user}] + preload: [owner: o, memberships: :user] ) end @@ -165,11 +164,14 @@ defmodule Plausible.SiteAdmin do owner = site.owner if owner do + escaped_name = Phoenix.HTML.html_escape(owner.name) |> Phoenix.HTML.safe_to_string() + escaped_email = Phoenix.HTML.html_escape(owner.email) |> Phoenix.HTML.safe_to_string() + {:safe, """ - #{owner.name} + #{escaped_name}

- #{owner.email} + #{escaped_email} """} end end diff --git a/lib/plausible/site/memberships.ex b/lib/plausible/site/memberships.ex index f277e277afb8..3e520b7e813f 100644 --- a/lib/plausible/site/memberships.ex +++ b/lib/plausible/site/memberships.ex @@ -38,13 +38,16 @@ defmodule Plausible.Site.Memberships do ) end + @spec all_pending_ownerships(String.t()) :: list() + def all_pending_ownerships(email) do + pending_ownership_invitation_q(email) + |> Repo.all() + end + @spec pending_ownerships?(String.t()) :: boolean() def pending_ownerships?(email) do - Repo.exists?( - from(i in Plausible.Auth.Invitation, - where: i.email == ^email and i.role == ^:owner - ) - ) + pending_ownership_invitation_q(email) + |> Repo.exists?() end @spec any_or_pending?(Plausible.Auth.User.t()) :: boolean() @@ -61,4 +64,10 @@ defmodule Plausible.Site.Memberships do ) |> Repo.exists?() end + + defp pending_ownership_invitation_q(email) do + from(i in Plausible.Auth.Invitation, + where: i.email == ^email and i.role == ^:owner + ) + end end diff --git a/lib/plausible/site/memberships/accept_invitation.ex b/lib/plausible/site/memberships/accept_invitation.ex index bffe63105e08..98f82d3ddfd8 100644 --- a/lib/plausible/site/memberships/accept_invitation.ex +++ b/lib/plausible/site/memberships/accept_invitation.ex @@ -26,7 +26,7 @@ defmodule Plausible.Site.Memberships.AcceptInvitation do @spec transfer_ownership(Site.t(), Auth.User.t()) :: {:ok, Site.Membership.t()} | {:error, - Billing.Quota.over_limits_error() + Billing.Quota.Limits.over_limits_error() | Ecto.Changeset.t() | :transfer_to_self | :no_plan} @@ -54,7 +54,7 @@ defmodule Plausible.Site.Memberships.AcceptInvitation do {:ok, Site.Membership.t()} | {:error, :invitation_not_found - | Billing.Quota.over_limits_error() + | Billing.Quota.Limits.over_limits_error() | Ecto.Changeset.t() | :no_plan} def accept_invitation(invitation_id, user) do diff --git a/lib/plausible/site/memberships/create_invitation.ex b/lib/plausible/site/memberships/create_invitation.ex index 4961f8f68e8a..69b4cbd8321d 100644 --- a/lib/plausible/site/memberships/create_invitation.ex +++ b/lib/plausible/site/memberships/create_invitation.ex @@ -41,7 +41,7 @@ defmodule Plausible.Site.Memberships.CreateInvitation do {:ok, [Membership.t()]} | {:error, invite_error() - | Quota.over_limits_error()} + | Quota.Limits.over_limits_error()} def bulk_transfer_ownership_direct(sites, new_owner) do Plausible.Repo.transaction(fn -> for site <- sites do @@ -134,8 +134,8 @@ defmodule Plausible.Site.Memberships.CreateInvitation do defp check_team_member_limit(site, _role, invitee_email) do site = Plausible.Repo.preload(site, :owner) - limit = Quota.team_member_limit(site.owner) - usage = Quota.team_member_usage(site.owner, exclude_emails: invitee_email) + limit = Quota.Limits.team_member_limit(site.owner) + usage = Quota.Usage.team_member_usage(site.owner, exclude_emails: [invitee_email]) if Quota.below_limit?(usage, limit), do: :ok, diff --git a/lib/plausible/site/memberships/invitations.ex b/lib/plausible/site/memberships/invitations.ex index c8569aad3466..a333ea129a07 100644 --- a/lib/plausible/site/memberships/invitations.ex +++ b/lib/plausible/site/memberships/invitations.ex @@ -66,7 +66,7 @@ defmodule Plausible.Site.Memberships.Invitations do on_ee do @spec ensure_can_take_ownership(Site.t(), Auth.User.t()) :: - :ok | {:error, Quota.over_limits_error() | :no_plan} + :ok | {:error, Quota.Limits.over_limits_error() | :no_plan} def ensure_can_take_ownership(site, new_owner) do site = Repo.preload(site, :owner) new_owner = Plausible.Users.with_subscription(new_owner) @@ -75,32 +75,13 @@ defmodule Plausible.Site.Memberships.Invitations do active_subscription? = Plausible.Billing.Subscriptions.active?(new_owner.subscription) if active_subscription? && plan != :free_10k do - usage_after_transfer = %{ - monthly_pageviews: monthly_pageview_usage_after_transfer(site, new_owner), - team_members: team_member_usage_after_transfer(site, new_owner), - sites: Quota.site_usage(new_owner) + 1 - } - - Quota.ensure_within_plan_limits(usage_after_transfer, plan) + new_owner + |> Quota.Usage.usage(pending_ownership_site_ids: [site.id]) + |> Quota.ensure_within_plan_limits(plan) else {:error, :no_plan} end end - - defp team_member_usage_after_transfer(site, new_owner) do - current_usage = Quota.team_member_usage(new_owner) - site_usage = Quota.team_member_usage(site.owner, site: site) - - extra_usage = - if Plausible.Sites.is_member?(new_owner.id, site), do: 0, else: 1 - - current_usage + site_usage + extra_usage - end - - defp monthly_pageview_usage_after_transfer(site, new_owner) do - site_ids = Plausible.Sites.owned_site_ids(new_owner) ++ [site.id] - Quota.monthly_pageview_usage(new_owner, site_ids) - end else @spec ensure_can_take_ownership(Site.t(), Auth.User.t()) :: :ok def ensure_can_take_ownership(_site, _new_owner) do @@ -116,8 +97,7 @@ defmodule Plausible.Site.Memberships.Invitations do def check_feature_access(site, new_owner, false = _selfhost?) do missing_features = - site - |> Quota.features_usage() + Quota.Usage.features_usage(nil, [site.id]) |> Enum.filter(&(&1.check_availability(new_owner) != :ok)) if missing_features == [] do diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex index eb792946ff63..4fbf857abb33 100644 --- a/lib/plausible/sites.ex +++ b/lib/plausible/sites.ex @@ -169,7 +169,25 @@ defmodule Plausible.Sites do def create(user, params) do with :ok <- Quota.ensure_can_add_new_site(user) do Ecto.Multi.new() - |> Ecto.Multi.insert(:site, Site.new(params)) + |> Ecto.Multi.put(:site_changeset, Site.new(params)) + |> Ecto.Multi.run(:clear_changed_from, fn + _repo, %{site_changeset: %{changes: %{domain: domain}}} -> + case get_for_user(user.id, domain, [:owner]) do + %Site{domain_changed_from: ^domain} = site -> + site + |> Ecto.Changeset.change() + |> Ecto.Changeset.put_change(:domain_changed_from, nil) + |> Ecto.Changeset.put_change(:domain_changed_at, nil) + |> Repo.update() + + _ -> + {:ok, :ignore} + end + + _repo, _context -> + {:ok, :ignore} + end) + |> Ecto.Multi.insert(:site, fn %{site_changeset: site} -> site end) |> Ecto.Multi.insert(:site_membership, fn %{site: site} -> Site.Membership.new(site, user) end) diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.ex index 42ee7b47c2ad..ffa59389ea6a 100644 --- a/lib/plausible/stats.ex +++ b/lib/plausible/stats.ex @@ -1,12 +1,16 @@ defmodule Plausible.Stats do use Plausible + alias Plausible.Stats.QueryResult + use Plausible.ClickhouseRepo alias Plausible.Stats.{ Breakdown, Aggregate, Timeseries, CurrentVisitors, - FilterSuggestions + FilterSuggestions, + QueryOptimizer, + SQL } use Plausible.DebugReplayInfo @@ -31,6 +35,15 @@ defmodule Plausible.Stats do CurrentVisitors.current_visitors(site) end + def query(site, query) do + optimized_query = QueryOptimizer.optimize(query) + + optimized_query + |> SQL.QueryBuilder.build(site) + |> ClickhouseRepo.all() + |> QueryResult.from(optimized_query) + end + on_ee do def funnel(site, query, funnel) do include_sentry_replay_info() diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 01eef54e638e..9a2e65bab607 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -1,7 +1,7 @@ defmodule Plausible.Stats.Aggregate do use Plausible.ClickhouseRepo use Plausible - import Plausible.Stats.{Base, Imported} + import Plausible.Stats.Base import Ecto.Query alias Plausible.Stats.{Query, Util} @@ -15,23 +15,21 @@ defmodule Plausible.Stats.Aggregate do Query.trace(query, metrics) - {event_metrics, session_metrics, other_metrics} = - metrics - |> Util.maybe_add_visitors_metric() - |> Plausible.Stats.TableDecider.partition_metrics(query) + query_with_metrics = %Query{query | metrics: metrics} - event_task = fn -> aggregate_events(site, query, event_metrics) end - - session_task = fn -> aggregate_sessions(site, query, session_metrics) end + q = Plausible.Stats.SQL.QueryBuilder.build(query_with_metrics, site) time_on_page_task = - if :time_on_page in other_metrics do + if :time_on_page in query_with_metrics.metrics do fn -> aggregate_time_on_page(site, query) end else fn -> %{} end end - Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task]) + Plausible.ClickhouseRepo.parallel_tasks([ + run_query_task(q), + time_on_page_task + ]) |> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end) |> Util.keep_requested_metrics(metrics) |> cast_revenue_metrics_to_money(currency) @@ -40,24 +38,8 @@ defmodule Plausible.Stats.Aggregate do |> Enum.into(%{}) end - defp aggregate_events(_, _, []), do: %{} - - defp aggregate_events(site, query, metrics) do - from(e in base_event_query(site, query), select: ^select_event_metrics(metrics)) - |> merge_imported(site, query, metrics) - |> maybe_add_conversion_rate(site, query, metrics) - |> ClickhouseRepo.one() - end - - defp aggregate_sessions(_, _, []), do: %{} - - defp aggregate_sessions(site, query, metrics) do - from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query)) - |> filter_converted_sessions(site, query) - |> merge_imported(site, query, metrics) - |> ClickhouseRepo.one() - |> Util.keep_requested_metrics(metrics) - end + defp run_query_task(nil), do: fn -> %{} end + defp run_query_task(q), do: fn -> ClickhouseRepo.one(q) end defp aggregate_time_on_page(site, query) do windowed_pages_q = diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 85ffa7cd4ce3..5354833a1f63 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -31,7 +31,7 @@ defmodule Plausible.Stats.Base do end end - defp query_events(site, query) do + def query_events(site, query) do q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query)) on_ee do @@ -62,14 +62,21 @@ defmodule Plausible.Stats.Base do pageviews: dynamic( [e], - fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name) + selected_as( + fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name), + :pageviews + ) ) } end defp select_event_metric(:events) do %{ - events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))")) + events: + dynamic( + [], + selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events) + ) } end @@ -82,7 +89,13 @@ defmodule Plausible.Stats.Base do defp select_event_metric(:visits) do %{ visits: - dynamic([e], fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id)) + dynamic( + [e], + selected_as( + fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id), + :visits + ) + ) } end @@ -108,6 +121,7 @@ defmodule Plausible.Stats.Base do defp select_event_metric(:percentage), do: %{} defp select_event_metric(:conversion_rate), do: %{} + defp select_event_metric(:group_conversion_rate), do: %{} defp select_event_metric(:total_visitors), do: %{} defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}") @@ -127,10 +141,13 @@ defmodule Plausible.Stats.Base do bounce_rate: dynamic( [], - fragment( - "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", - ^condition, - ^condition + selected_as( + fragment( + "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", + ^condition, + ^condition + ), + :bounce_rate ) ), __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) @@ -139,7 +156,14 @@ defmodule Plausible.Stats.Base do defp select_session_metric(:visits, _query) do %{ - visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)) + visits: + dynamic( + [s], + selected_as( + fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign), + :visits + ) + ) } end @@ -148,7 +172,10 @@ defmodule Plausible.Stats.Base do pageviews: dynamic( [s], - fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews) + selected_as( + fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews), + :pageviews + ) ) } end @@ -158,7 +185,10 @@ defmodule Plausible.Stats.Base do events: dynamic( [s], - fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events) + selected_as( + fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events), + :events + ) ) } end @@ -179,7 +209,13 @@ defmodule Plausible.Stats.Base do defp select_session_metric(:visit_duration, _query) do %{ visit_duration: - dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")), + dynamic( + [], + selected_as( + fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"), + :visit_duration + ) + ), __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) } end @@ -189,7 +225,15 @@ defmodule Plausible.Stats.Base do views_per_visit: dynamic( [s], - fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign) + selected_as( + fragment( + "ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", + s.sign, + s.pageviews, + s.sign + ), + :views_per_visit + ) ), __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) } @@ -206,6 +250,8 @@ defmodule Plausible.Stats.Base do end defp select_session_metric(:percentage, _query), do: %{} + defp select_session_metric(:conversion_rate, _query), do: %{} + defp select_session_metric(:group_conversion_rate, _query), do: %{} def filter_converted_sessions(db_query, site, query) do if Query.has_event_filters?(query) do @@ -301,9 +347,9 @@ defmodule Plausible.Stats.Base do # only if it's included in the base query - otherwise the total will be based on # a different data set, making the metric inaccurate. This is why we're using an # explicit `include_imported` argument here. - defp total_visitors_subquery(site, query, include_imported) + def total_visitors_subquery(site, query, include_imported) - defp total_visitors_subquery(site, query, true = _include_imported) do + def total_visitors_subquery(site, query, true = _include_imported) do dynamic( [e], selected_as( @@ -314,13 +360,13 @@ defmodule Plausible.Stats.Base do ) end - defp total_visitors_subquery(site, query, false = _include_imported) do + def total_visitors_subquery(site, query, false = _include_imported) do dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors)) end def add_percentage_metric(q, site, query, metrics) do if :percentage in metrics do - total_query = Query.set_property(query, nil) + total_query = Query.set_dimensions(query, []) q |> select_merge( @@ -328,11 +374,14 @@ defmodule Plausible.Stats.Base do ) |> select_merge(%{ percentage: - fragment( - "if(? > 0, round(? / ? * 100, 1), null)", - selected_as(:__total_visitors), - selected_as(:visitors), - selected_as(:__total_visitors) + selected_as( + fragment( + "if(? > 0, round(? / ? * 100, 1), null)", + selected_as(:__total_visitors), + selected_as(:visitors), + selected_as(:__total_visitors) + ), + :percentage ) }) else @@ -348,7 +397,7 @@ defmodule Plausible.Stats.Base do total_query = query |> Query.remove_filters(["event:goal", "event:props"]) - |> Query.set_property(nil) + |> Query.set_dimensions([]) # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL subquery(q) @@ -357,11 +406,14 @@ defmodule Plausible.Stats.Base do ) |> select_merge([e], %{ conversion_rate: - fragment( - "if(? > 0, round(? / ? * 100, 1), 0)", - selected_as(:__total_visitors), - e.visitors, - selected_as(:__total_visitors) + selected_as( + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + selected_as(:__total_visitors), + e.visitors, + selected_as(:__total_visitors) + ), + :conversion_rate ) }) else diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 23db74a9e04c..a0bdaa150917 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -3,344 +3,63 @@ defmodule Plausible.Stats.Breakdown do use Plausible use Plausible.Stats.Fragments - import Plausible.Stats.{Base, Imported} + import Plausible.Stats.Base import Ecto.Query - alias Plausible.Stats.{Query, Util, TableDecider} - - @no_ref "Direct / None" - @not_set "(not set)" - - @session_metrics [:bounce_rate, :visit_duration] - - @revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) - - @event_metrics [:visits, :visitors, :pageviews, :events, :percentage] ++ @revenue_metrics - - # These metrics can be asked from the `breakdown/5` function, - # but they are different from regular metrics such as `visitors`, - # or `bounce_rate` - we cannot currently "select them" directly in - # the db queries. Instead, we need to artificially append them to - # the breakdown results later on. - @computed_metrics [:conversion_rate, :total_visitors] - - def breakdown(site, query, metrics, pagination, opts \\ []) - - def breakdown( - site, - %Query{property: "event:goal"} = query, - metrics, - pagination, - opts - ) do - site = Plausible.Repo.preload(site, :goals) - - {event_goals, pageview_goals} = Enum.split_with(site.goals, & &1.event_name) - events = Enum.map(event_goals, & &1.event_name) - - event_query = - query - |> Query.put_filter([:member, "event:name", events]) - |> Query.set_property("event:name") - - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) - - no_revenue = {nil, metrics -- @revenue_metrics} - - {revenue_goals, metrics} = - on_ee do - if Plausible.Billing.Feature.RevenueGoals.enabled?(site) do - revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1) - metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics - - {revenue_goals, metrics} - else - no_revenue - end - else - no_revenue - end - - metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - - event_q = - if Enum.any?(event_goals) do - site - |> breakdown_events(event_query, metrics_to_select) - |> apply_pagination(pagination) - else - nil - end - - page_q = - if Enum.any?(pageview_goals) do - page_query = Query.set_property(query, "event:page") - - page_exprs = Enum.map(pageview_goals, & &1.page_path) - page_regexes = Enum.map(page_exprs, &page_regex/1) - - select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil - - from(e in base_event_query(site, page_query), - order_by: [desc: fragment("uniq(?)", e.user_id)], - where: - fragment( - "notEmpty(multiMatchAllIndices(?, ?) as indices)", - e.pathname, - ^page_regexes - ) and e.name == "pageview", - array_join: index in fragment("indices"), - group_by: index, - select: %{ - name: fragment("concat('Visit ', ?[?])", ^page_exprs, index) - } - ) - |> select_merge(^select_columns) - |> merge_imported_pageview_goals(site, page_query, page_exprs, metrics_to_select) - |> apply_pagination(pagination) - else - nil - end - - full_q = - case {event_q, page_q} do - {nil, nil} -> - nil - - {event_q, nil} -> - event_q - - {nil, page_q} -> - page_q - - {event_q, page_q} -> - from( - e in subquery(union_all(event_q, ^page_q)), - order_by: [desc: e.visitors] - ) - |> apply_pagination(pagination) - end - - if full_q do - full_q - |> maybe_add_conversion_rate(site, query, metrics) - |> ClickhouseRepo.all() - |> transform_keys(%{name: :goal}) - |> cast_revenue_metrics_to_money(revenue_goals) - |> Util.keep_requested_metrics(metrics) - else - [] - end - end - - def breakdown( - site, - %Query{property: "event:props:" <> custom_prop} = query, - metrics, - pagination, - opts - ) do - {currency, metrics} = - on_ee do - Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, metrics) - else - {nil, metrics} - end - - metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) - - breakdown_events(site, query, metrics_to_select) - |> maybe_add_conversion_rate(site, query, metrics) - |> paginate_and_execute(metrics, pagination) - |> transform_keys(%{breakdown_prop_value: custom_prop}) - |> Enum.map(&cast_revenue_metrics_to_money(&1, currency)) - end - - def breakdown(site, %Query{property: "event:page"} = query, metrics, pagination, opts) do - event_metrics = - metrics - |> Util.maybe_add_visitors_metric() - |> Enum.filter(&(&1 in @event_metrics)) - - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) - - event_result = - site - |> breakdown_events(query, event_metrics) - |> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics) - |> paginate_and_execute(metrics, pagination) - |> maybe_add_time_on_page(site, query, metrics) - - session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) - - entry_page_query = - case event_result do - [] -> - query - - pages -> - query - |> Query.remove_filters(["event:page"]) - |> Query.put_filter([:member, "visit:entry_page", Enum.map(pages, & &1[:page])]) - |> Query.set_property("visit:entry_page") - end - - if Enum.any?(event_metrics) && Enum.empty?(event_result) do - [] - else - {limit, _page} = pagination - - session_result = - breakdown_sessions(site, entry_page_query, session_metrics) - |> paginate_and_execute(session_metrics, {limit, 1}) - |> transform_keys(%{entry_page: :page}) - - metrics = metrics ++ [:page] - - zip_results( - event_result, - session_result, - :page, - metrics + alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL} + alias Plausible.Stats.Filters.QueryParser + + def breakdown(site, %Query{dimensions: [dimension]} = query, metrics, pagination, _opts \\ []) do + transformed_metrics = transform_metrics(metrics, dimension) + + query_with_metrics = + Query.set( + query, + metrics: transformed_metrics, + order_by: infer_order_by(transformed_metrics, dimension), + dimensions: transform_dimensions(dimension), + filters: query.filters ++ dimension_filters(dimension), + preloaded_goals: QueryParser.preload_goals_if_needed(site, query.filters, [dimension]), + v2: true, + # Allow pageview and event metrics to be queried off of sessions table + legacy_breakdown: true ) - |> Enum.map(&Map.take(&1, metrics)) - end - end - - def breakdown(site, %Query{property: "event:name"} = query, metrics, pagination, opts) do - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) - - breakdown_events(site, query, metrics) - |> paginate_and_execute(metrics, pagination) - end - - def breakdown(site, query, metrics, pagination, opts) do - query = maybe_update_breakdown_filters(query) - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) - - metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - - case breakdown_table(query, metrics) do - :session -> - breakdown_sessions(site, query, metrics_to_select) - |> maybe_add_group_conversion_rate(&breakdown_sessions/3, site, query, metrics) - |> paginate_and_execute(metrics, pagination) - - :event -> - breakdown_events(site, query, metrics_to_select) - |> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics) - |> paginate_and_execute(metrics, pagination) - end - end - - defp maybe_update_breakdown_filters(%Query{property: visit_entry_prop} = query) - when visit_entry_prop in [ - "visit:source", - "visit:entry_page", - "visit:utm_medium", - "visit:utm_source", - "visit:utm_campaign", - "visit:utm_content", - "visit:utm_term", - "visit:entry_page", - "visit:referrer" - ] do - update_hostname_filter_prop(query, "visit:entry_page_hostname") - end - - defp maybe_update_breakdown_filters(%Query{property: "visit:exit_page"} = query) do - update_hostname_filter_prop(query, "visit:exit_page_hostname") - end - - defp maybe_update_breakdown_filters(query) do - query - end + |> QueryOptimizer.optimize() - defp update_hostname_filter_prop(query, visit_prop) do - case Query.get_filter(query, "event:hostname") do - nil -> - query + q = SQL.QueryBuilder.build(query_with_metrics, site) - [op, "event:hostname", value] -> - query - |> Query.put_filter([op, visit_prop, value]) - end - end - - # Backwards compatibility - defp breakdown_table(%Query{experimental_reduced_joins?: false}, _), do: :session - - defp breakdown_table(%Query{property: "visit:entry_page"}, _metrics), do: :session - defp breakdown_table(%Query{property: "visit:entry_page_hostname"}, _metrics), do: :session - defp breakdown_table(%Query{property: "visit:exit_page"}, _metrics), do: :session - defp breakdown_table(%Query{property: "visit:exit_page_hostname"}, _metrics), do: :session - - defp breakdown_table(%Query{property: property} = query, metrics) do - {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query, property) - - if not Enum.empty?(session_metrics) do - :session - else - :event - end + q + |> apply_pagination(pagination) + |> ClickhouseRepo.all() + |> QueryResult.from(query_with_metrics) + |> build_breakdown_result(query_with_metrics, metrics) + |> maybe_add_time_on_page(site, query_with_metrics, metrics) + |> update_currency_metrics(site, query_with_metrics) end - defp zip_results(event_result, session_result, property, metrics) do - null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{}) + defp build_breakdown_result(query_result, query, metrics) do + query_result.results + |> Enum.map(fn %{dimensions: dimensions, metrics: entry_metrics} -> + dimension_map = + query.dimensions |> Enum.map(&result_key/1) |> Enum.zip(dimensions) |> Enum.into(%{}) - prop_values = - Enum.map(event_result ++ session_result, fn row -> row[property] end) - |> Enum.uniq() + metrics_map = Enum.zip(metrics, entry_metrics) |> Enum.into(%{}) - Enum.map(prop_values, fn value -> - event_row = Enum.find(event_result, fn row -> row[property] == value end) || %{} - session_row = Enum.find(session_result, fn row -> row[property] == value end) || %{} - - null_row - |> Map.merge(event_row) - |> Map.merge(session_row) + Map.merge(dimension_map, metrics_map) end) - |> sort_results(metrics) - end - - defp breakdown_sessions(site, %Query{property: property} = query, metrics) do - from(s in query_sessions(site, query), - order_by: [desc: fragment("uniq(?)", s.user_id)], - select: ^select_session_metrics(metrics, query) - ) - |> filter_converted_sessions(site, query) - |> do_group_by(property) - |> merge_imported(site, query, metrics) - |> add_percentage_metric(site, query, metrics) - end - - defp breakdown_events(site, %Query{property: property} = query, metrics) do - from(e in base_event_query(site, query), - order_by: [desc: fragment("uniq(?)", e.user_id)], - select: %{} - ) - |> do_group_by(property) - |> select_merge(^select_event_metrics(metrics)) - |> merge_imported(site, query, metrics) - |> add_percentage_metric(site, query, metrics) end - defp paginate_and_execute(_, [], _), do: [] - - defp paginate_and_execute(q, metrics, pagination) do - q - |> apply_pagination(pagination) - |> ClickhouseRepo.all() - |> Util.keep_requested_metrics(metrics) - end + defp result_key("event:props:" <> custom_property), do: custom_property + defp result_key("event:" <> key), do: key |> String.to_existing_atom() + defp result_key("visit:" <> key), do: key |> String.to_existing_atom() + defp result_key(dimension), do: dimension defp maybe_add_time_on_page(event_results, site, query, metrics) do - if :time_on_page in metrics do + if query.dimensions == ["event:page"] and :time_on_page in metrics do pages = Enum.map(event_results, & &1[:page]) time_on_page_result = breakdown_time_on_page(site, query, pages) - Enum.map(event_results, fn row -> + event_results + |> Enum.map(fn row -> Map.put(row, :time_on_page, time_on_page_result[row[:page]]) end) else @@ -431,359 +150,102 @@ defmodule Plausible.Stats.Breakdown do |> Map.new() end - defp do_group_by( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q, - "event:props:" <> prop - ) do - from( - e in q, - select_merge: %{ - breakdown_prop_value: - selected_as( - fragment( - "if(not empty(?), ?, '(none)')", - get_by_key(e, :meta, ^prop), - get_by_key(e, :meta, ^prop) - ), - :breakdown_prop_value - ) - }, - group_by: selected_as(:breakdown_prop_value), - order_by: {:asc, selected_as(:breakdown_prop_value)} - ) - end - - defp do_group_by( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q, - "event:name" - ) do - from( - e in q, - group_by: e.name, - select_merge: %{name: e.name}, - order_by: {:asc, e.name} - ) - end - - defp do_group_by( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q, - "event:page" - ) do - from( - e in q, - group_by: e.pathname, - select_merge: %{page: e.pathname}, - order_by: {:asc, e.pathname} - ) - end - - defp do_group_by(q, "visit:source") do - from( - s in q, - group_by: s.source, - select_merge: %{ - source: fragment("if(empty(?), ?, ?)", s.source, @no_ref, s.source) - }, - order_by: {:asc, s.source} - ) - end - - defp do_group_by(q, "visit:country") do - from( - s in q, - where: s.country != "\0\0" and s.country != "ZZ", - group_by: s.country, - select_merge: %{country: s.country}, - order_by: {:asc, s.country} - ) - end - - defp do_group_by(q, "visit:region") do - from( - s in q, - where: s.region != "", - group_by: s.region, - select_merge: %{region: s.region}, - order_by: {:asc, s.region} - ) - end - - defp do_group_by(q, "visit:city") do - from( - s in q, - where: s.city != 0, - group_by: s.city, - select_merge: %{city: s.city}, - order_by: {:asc, s.city} - ) - end - - defp do_group_by(q, "visit:entry_page") do - from( - s in q, - # Sessions without pageviews don't get entry_page assigned, hence they should get ignored - where: s.entry_page != "", - group_by: s.entry_page, - select_merge: %{entry_page: s.entry_page}, - order_by: {:asc, s.entry_page} - ) - end - - defp do_group_by(q, "visit:exit_page") do - from( - s in q, - # Sessions without pageviews don't get entry_page assigned, hence they should get ignored - where: s.entry_page != "", - group_by: s.exit_page, - select_merge: %{exit_page: s.exit_page}, - order_by: {:asc, s.exit_page} - ) - end - - defp do_group_by(q, "visit:referrer") do - from( - s in q, - group_by: s.referrer, - select_merge: %{ - referrer: fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer) - }, - order_by: {:asc, s.referrer} - ) - end - - defp do_group_by(q, "visit:utm_medium") do - from( - s in q, - where: fragment("not empty(?)", s.utm_medium), - group_by: s.utm_medium, - select_merge: %{ - utm_medium: s.utm_medium - } - ) - end + defp transform_metrics(metrics, dimension) do + metrics = + if is_nil(metric_to_order_by(metrics)) do + metrics ++ [:visitors] + else + metrics + end - defp do_group_by(q, "visit:utm_source") do - from( - s in q, - where: fragment("not empty(?)", s.utm_source), - group_by: s.utm_source, - select_merge: %{ - utm_source: s.utm_source - } - ) + Enum.map(metrics, fn metric -> + case {metric, dimension} do + {:conversion_rate, "event:props:" <> _} -> :conversion_rate + {:conversion_rate, "event:goal"} -> :conversion_rate + {:conversion_rate, _} -> :group_conversion_rate + _ -> metric + end + end) end - defp do_group_by(q, "visit:utm_campaign") do - from( - s in q, - where: fragment("not empty(?)", s.utm_campaign), - group_by: s.utm_campaign, - select_merge: %{ - utm_campaign: s.utm_campaign - } - ) - end + defp infer_order_by(metrics, "event:goal"), do: [{metric_to_order_by(metrics), :desc}] - defp do_group_by(q, "visit:utm_content") do - from( - s in q, - where: fragment("not empty(?)", s.utm_content), - group_by: s.utm_content, - select_merge: %{ - utm_content: s.utm_content - } - ) - end + defp infer_order_by(metrics, dimension), + do: [{metric_to_order_by(metrics), :desc}, {dimension, :asc}] - defp do_group_by(q, "visit:utm_term") do - from( - s in q, - where: fragment("not empty(?)", s.utm_term), - group_by: s.utm_term, - select_merge: %{ - utm_term: s.utm_term - } - ) + defp metric_to_order_by(metrics) do + Enum.find(metrics, &(&1 != :time_on_page)) end - defp do_group_by(q, "visit:device") do - from( - s in q, - group_by: s.device, - select_merge: %{ - device: fragment("if(empty(?), ?, ?)", s.device, @not_set, s.device) - }, - order_by: {:asc, s.device} - ) - end + def transform_dimensions("visit:browser_version"), + do: ["visit:browser", "visit:browser_version"] - defp do_group_by(q, "visit:os") do - from( - s in q, - group_by: s.os, - select_merge: %{ - os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os) - }, - order_by: {:asc, s.os} - ) - end + def transform_dimensions("visit:os_version"), do: ["visit:os", "visit:os_version"] + def transform_dimensions(dimension), do: [dimension] - defp do_group_by(q, "visit:os_version") do - from( - s in q, - group_by: [s.os, s.os_version], - select_merge: %{ - os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os), - os_version: - fragment( - "if(empty(?), ?, ?)", - s.os_version, - @not_set, - s.os_version - ) - }, - order_by: {:asc, s.os_version} - ) - end + @filter_dimensions_not %{ + "visit:city" => [0], + "visit:country" => ["\0\0", "ZZ"], + "visit:region" => [""], + "visit:utm_medium" => [""], + "visit:utm_source" => [""], + "visit:utm_campaign" => [""], + "visit:utm_content" => [""], + "visit:utm_term" => [""], + "visit:entry_page" => [""], + "visit:exit_page" => [""] + } - defp do_group_by(q, "visit:browser") do - from( - s in q, - group_by: s.browser, - select_merge: %{ - browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser) - }, - order_by: {:asc, s.browser} - ) - end + @extra_filter_dimensions Map.keys(@filter_dimensions_not) - defp do_group_by(q, "visit:browser_version") do - from( - s in q, - group_by: [s.browser, s.browser_version], - select_merge: %{ - browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser), - browser_version: - fragment("if(empty(?), ?, ?)", s.browser_version, @not_set, s.browser_version) - }, - order_by: {:asc, s.browser_version} - ) + defp dimension_filters(dimension) when dimension in @extra_filter_dimensions do + [[:is_not, dimension, Map.get(@filter_dimensions_not, dimension)]] end - defp group_by_field_names("event:props:" <> _prop), do: [:name] - defp group_by_field_names("visit:os_version"), do: [:os, :os_version] - defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version] + defp dimension_filters(_), do: [] - defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)] + defp apply_pagination(q, {limit, page}) do + offset = (page - 1) * limit - defp on_matches_group_by(fields) do - Enum.reduce(fields, nil, &fields_equal/2) + q + |> limit(^limit) + |> offset(^offset) end - defp outer_order_by(fields) do - Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end) - end + on_ee do + defp update_currency_metrics(results, site, %Query{dimensions: ["event:goal"]}) do + site = Plausible.Repo.preload(site, :goals) - defp fields_equal(field_name, nil), - do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name)) - - defp fields_equal(field_name, condition), - do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition) - - defp sort_results(results, metrics) do - Enum.sort_by( - results, - fn entry -> - case entry[sorting_key(metrics)] do - nil -> 0 - n -> n - end - end, - :desc - ) - end + {event_goals, _pageview_goals} = Enum.split_with(site.goals, & &1.event_name) + revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1) - # This function injects a conversion_rate metric into - # a breakdown query. It is calculated as X / Y, where: - # - # * X is the number of conversions for a breakdown - # result (conversion = number of visitors who - # completed the filtered goal with the filtered - # custom properties). - # - # * Y is the number of all visitors for this breakdown - # result without the `event:goal` and `event:props:*` - # filters. - defp maybe_add_group_conversion_rate( - q, - breakdown_fn, - site, - %Query{property: property} = query, - metrics - ) do - if :conversion_rate in metrics do - breakdown_total_visitors_query = - query |> Query.remove_filters(["event:goal", "event:props"]) - - breakdown_total_visitors_q = - breakdown_fn.(site, breakdown_total_visitors_query, [:visitors]) - - from(e in subquery(q), - left_join: c in subquery(breakdown_total_visitors_q), - on: ^on_matches_group_by(group_by_field_names(property)), - select_merge: %{ - total_visitors: c.visitors, - conversion_rate: - fragment( - "if(? > 0, round(? / ? * 100, 1), 0)", - c.visitors, - e.visitors, - c.visitors - ) - }, - order_by: [desc: e.visitors], - order_by: ^outer_order_by(group_by_field_names(property)) - ) - else - q + if length(revenue_goals) > 0 and Plausible.Billing.Feature.RevenueGoals.enabled?(site) do + Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals) + else + remove_revenue_metrics(results) + end end - end - # When querying custom event goals and pageviewgoals together, UNION ALL is used - # so the same fields must be present on both sides of the union. This change to the - # query will ensure that we don't unnecessarily read revenue column for pageview goals - defp mark_revenue_as_nil(select_columns) do - select_columns - |> Map.replace(:total_revenue, nil) - |> Map.replace(:average_revenue, nil) - end + defp update_currency_metrics(results, site, query) do + {currency, _metrics} = + Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, query.metrics) - defp sorting_key(metrics) do - if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics) + if currency do + results + |> Enum.map(&Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(&1, currency)) + else + remove_revenue_metrics(results) + end + end + else + defp update_currency_metrics(results, _site, _query), do: remove_revenue_metrics(results) end - defp transform_keys(results, keys_to_replace) do + defp remove_revenue_metrics(results) do Enum.map(results, fn map -> - Enum.map(map, fn {key, val} -> - {Map.get(keys_to_replace, key, key), val} - end) - |> Enum.into(%{}) + map + |> Map.delete(:total_revenue) + |> Map.delete(:average_revenue) end) end - - defp apply_pagination(q, {limit, page}) do - offset = (page - 1) * limit - - q - |> limit(^limit) - |> offset(^offset) - end - - on_ee do - defp cast_revenue_metrics_to_money(results, revenue_goals) do - Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals) - end - else - defp cast_revenue_metrics_to_money(results, _revenue_goals), do: results - end end diff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.ex index 675597731b34..1dc26465bb1b 100644 --- a/lib/plausible/stats/email_report.ex +++ b/lib/plausible/stats/email_report.ex @@ -40,7 +40,7 @@ defmodule Plausible.Stats.EmailReport do end defp put_top_5_pages(stats, site, query) do - query = Query.set_property(query, "event:page") + query = Query.set_dimensions(query, ["event:page"]) pages = Stats.breakdown(site, query, [:visitors], {5, 1}) Map.put(stats, :pages, pages) end @@ -48,8 +48,8 @@ defmodule Plausible.Stats.EmailReport do defp put_top_5_sources(stats, site, query) do query = query - |> Query.put_filter([:is_not, "visit:source", "Direct / None"]) - |> Query.set_property("visit:source") + |> Query.put_filter([:is_not, "visit:source", ["Direct / None"]]) + |> Query.set_dimensions(["visit:source"]) sources = Stats.breakdown(site, query, [:visitors], {5, 1}) diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex index 3775e734359f..b6760965edf7 100644 --- a/lib/plausible/stats/filters/filters.ex +++ b/lib/plausible/stats/filters/filters.ex @@ -3,7 +3,8 @@ defmodule Plausible.Stats.Filters do A module for parsing filters used in stat queries. """ - alias Plausible.Stats.Filters.{DashboardFilterParser, StatsAPIFilterParser} + alias Plausible.Stats.Filters.QueryParser + alias Plausible.Stats.Filters.{LegacyDashboardFilterParser, StatsAPIFilterParser} @visit_props [ :source, @@ -47,32 +48,39 @@ defmodule Plausible.Stats.Filters do Depending on the format and type of the `filters` argument, returns: - * a decoded map, when `filters` is encoded JSON - * a parsed filter map, when `filters` is a filter expression string - * the same map, when `filters` is a map + * a decoded list, when `filters` is encoded JSON + * a parsed filter list, when `filters` is a filter expression string + * the same list, when `filters` is a map - Returns an empty map when argument type is unexpected (e.g. `nil`). + Returns an empty list when argument type is unexpected (e.g. `nil`). ### Examples: iex> Filters.parse("{\\"page\\":\\"/blog/**\\"}") - [[:matches, "event:page", "/blog/**"]] + [[:matches, "event:page", ["/blog/**"]]] iex> Filters.parse("visit:browser!=Chrome") - [[:is_not, "visit:browser", "Chrome"]] + [[:is_not, "visit:browser", ["Chrome"]]] iex> Filters.parse(nil) [] """ def parse(filters) when is_binary(filters) do case Jason.decode(filters) do - {:ok, filters} when is_map(filters) -> DashboardFilterParser.parse_and_prefix(filters) + {:ok, filters} when is_map(filters) or is_list(filters) -> parse(filters) {:ok, _} -> [] {:error, err} -> StatsAPIFilterParser.parse_filter_expression(err.data) end end - def parse(filters) when is_map(filters), do: DashboardFilterParser.parse_and_prefix(filters) + def parse(filters) when is_map(filters), + do: LegacyDashboardFilterParser.parse_and_prefix(filters) + + def parse(filters) when is_list(filters) do + {:ok, parsed_filters} = QueryParser.parse_filters(filters) + parsed_filters + end + def parse(_), do: [] def without_prefix(property) do diff --git a/lib/plausible/stats/filters/dashboard_filter_parser.ex b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex similarity index 82% rename from lib/plausible/stats/filters/dashboard_filter_parser.ex rename to lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex index 926642eac4e5..67e8c8cd0f89 100644 --- a/lib/plausible/stats/filters/dashboard_filter_parser.ex +++ b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex @@ -1,4 +1,4 @@ -defmodule Plausible.Stats.Filters.DashboardFilterParser do +defmodule Plausible.Stats.Filters.LegacyDashboardFilterParser do @moduledoc false import Plausible.Stats.Filters.Utils @@ -36,40 +36,40 @@ defmodule Plausible.Stats.Filters.DashboardFilterParser do cond do is_negated && is_wildcard && is_list -> - [:not_matches_member, key, val] + [:does_not_match, key, val] is_negated && is_contains && is_list -> - [:not_matches_member, key, Enum.map(val, &"**#{&1}**")] + [:does_not_match, key, Enum.map(val, &"**#{&1}**")] is_wildcard && is_list -> - [:matches_member, key, val] + [:matches, key, val] is_negated && is_wildcard -> - [:does_not_match, key, val] + [:does_not_match, key, [val]] is_negated && is_list -> - [:not_member, key, val] + [:is_not, key, val] is_negated && is_contains -> - [:does_not_match, key, "**" <> val <> "**"] + [:does_not_match, key, ["**" <> val <> "**"]] is_contains && is_list -> - [:matches_member, key, Enum.map(val, &"**#{&1}**")] + [:matches, key, Enum.map(val, &"**#{&1}**")] is_negated -> - [:is_not, key, val] + [:is_not, key, [val]] is_list -> - [:member, key, val] + [:is, key, val] is_contains -> - [:matches, key, "**" <> val <> "**"] + [:matches, key, ["**" <> val <> "**"]] is_wildcard -> - [:matches, key, val] + [:matches, key, [val]] true -> - [:is, key, val] + [:is, key, [val]] end end diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex new file mode 100644 index 000000000000..3f802dcb64df --- /dev/null +++ b/lib/plausible/stats/filters/query_parser.ex @@ -0,0 +1,414 @@ +defmodule Plausible.Stats.Filters.QueryParser do + @moduledoc false + + alias Plausible.Stats.TableDecider + alias Plausible.Stats.Filters + alias Plausible.Stats.Query + alias Plausible.Stats.Metrics + + def parse(site, params, now \\ nil) when is_map(params) do + with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])), + {:ok, filters} <- parse_filters(Map.get(params, "filters", [])), + {:ok, date_range} <- + parse_date_range(site, Map.get(params, "date_range"), now || today(site)), + {:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])), + {:ok, order_by} <- parse_order_by(Map.get(params, "order_by")), + {:ok, include} <- parse_include(Map.get(params, "include", %{})), + preloaded_goals <- preload_goals_if_needed(site, filters, dimensions), + query = %{ + metrics: metrics, + filters: filters, + date_range: date_range, + dimensions: dimensions, + order_by: order_by, + timezone: site.timezone, + imported_data_requested: Map.get(include, :imports, false), + preloaded_goals: preloaded_goals + }, + :ok <- validate_order_by(query), + :ok <- validate_goal_filters(query), + :ok <- validate_custom_props_access(site, query), + :ok <- validate_metrics(query) do + {:ok, query} + end + end + + defp parse_metrics([]), do: {:error, "No valid metrics passed"} + + defp parse_metrics(metrics) when is_list(metrics) do + if length(metrics) == length(Enum.uniq(metrics)) do + parse_list(metrics, &parse_metric/1) + else + {:error, "Metrics cannot be queried multiple times"} + end + end + + defp parse_metrics(_invalid_metrics), do: {:error, "Invalid metrics passed"} + + defp parse_metric(metric_str) do + case Metrics.from_string(metric_str) do + {:ok, metric} -> {:ok, metric} + _ -> {:error, "Unknown metric '#{inspect(metric_str)}'"} + end + end + + def parse_filters(filters) when is_list(filters) do + parse_list(filters, &parse_filter/1) + end + + def parse_filters(_invalid_metrics), do: {:error, "Invalid filters passed"} + + defp parse_filter(filter) do + with {:ok, operator} <- parse_operator(filter), + {:ok, filter_key} <- parse_filter_key(filter), + {:ok, rest} <- parse_filter_rest(operator, filter) do + {:ok, [operator, filter_key | rest]} + end + end + + defp parse_operator(["is" | _rest]), do: {:ok, :is} + defp parse_operator(["is_not" | _rest]), do: {:ok, :is_not} + defp parse_operator(["matches" | _rest]), do: {:ok, :matches} + defp parse_operator(["does_not_match" | _rest]), do: {:ok, :does_not_match} + defp parse_operator(["contains" | _rest]), do: {:ok, :contains} + defp parse_operator(["does_not_contain" | _rest]), do: {:ok, :does_not_contain} + defp parse_operator(filter), do: {:error, "Unknown operator for filter '#{inspect(filter)}'"} + + defp parse_filter_key([_operator, filter_key | _rest] = filter) do + parse_filter_key_string(filter_key, "Invalid filter '#{inspect(filter)}") + end + + defp parse_filter_key(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"} + + defp parse_filter_rest(operator, filter) + when operator in [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain], + do: parse_clauses_list(filter) + + defp parse_clauses_list([_operation, filter_key, list] = filter) when is_list(list) do + all_strings? = Enum.all?(list, &is_binary/1) + + cond do + filter_key == "event:goal" && all_strings? -> {:ok, [Filters.Utils.wrap_goal_value(list)]} + filter_key != "event:goal" && all_strings? -> {:ok, [list]} + true -> {:error, "Invalid filter '#{inspect(filter)}'"} + end + end + + defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"} + + defp parse_date_range(_site, "day", date) do + {:ok, Date.range(date, date)} + end + + defp parse_date_range(_site, "7d", last) do + first = last |> Date.add(-6) + {:ok, Date.range(first, last)} + end + + defp parse_date_range(_site, "30d", last) do + first = last |> Date.add(-30) + {:ok, Date.range(first, last)} + end + + defp parse_date_range(_site, "month", today) do + last = today |> Date.end_of_month() + first = last |> Date.beginning_of_month() + {:ok, Date.range(first, last)} + end + + defp parse_date_range(_site, "6mo", today) do + last = today |> Date.end_of_month() + + first = + last + |> Timex.shift(months: -5) + |> Date.beginning_of_month() + + {:ok, Date.range(first, last)} + end + + defp parse_date_range(_site, "12mo", today) do + last = today |> Date.end_of_month() + + first = + last + |> Timex.shift(months: -11) + |> Date.beginning_of_month() + + {:ok, Date.range(first, last)} + end + + defp parse_date_range(_site, "year", today) do + last = today |> Timex.end_of_year() + first = last |> Timex.beginning_of_year() + {:ok, Date.range(first, last)} + end + + defp parse_date_range(site, "all", today) do + start_date = Plausible.Sites.stats_start_date(site) || today + + {:ok, Date.range(start_date, today)} + end + + defp parse_date_range(_site, [from_date_string, to_date_string], _date) + when is_binary(from_date_string) and is_binary(to_date_string) do + with {:ok, from_date} <- Date.from_iso8601(from_date_string), + {:ok, to_date} <- Date.from_iso8601(to_date_string) do + {:ok, Date.range(from_date, to_date)} + else + _ -> {:error, "Invalid date_range '#{inspect([from_date_string, to_date_string])}'"} + end + end + + defp parse_date_range(_site, unknown, _), + do: {:error, "Invalid date_range '#{inspect(unknown)}'"} + + defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date() + + defp parse_dimensions(dimensions) when is_list(dimensions) do + if length(dimensions) == length(Enum.uniq(dimensions)) do + parse_list( + dimensions, + &parse_dimension_entry(&1, "Invalid dimensions '#{inspect(dimensions)}'") + ) + else + {:error, "Some dimensions are listed multiple times"} + end + end + + defp parse_dimensions(dimensions), do: {:error, "Invalid dimensions '#{inspect(dimensions)}'"} + + defp parse_order_by(order_by) when is_list(order_by) do + parse_list(order_by, &parse_order_by_entry/1) + end + + defp parse_order_by(nil), do: {:ok, nil} + defp parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"} + + defp parse_order_by_entry(entry) do + with {:ok, value} <- parse_metric_or_dimension(entry), + {:ok, order_direction} <- parse_order_direction(entry) do + {:ok, {value, order_direction}} + end + end + + defp parse_dimension_entry(key, error_message) do + case { + parse_time(key), + parse_filter_key_string(key) + } do + {{:ok, time}, _} -> {:ok, time} + {_, {:ok, filter_key}} -> {:ok, filter_key} + _ -> {:error, error_message} + end + end + + defp parse_metric_or_dimension([value, _] = entry) do + case { + parse_time(value), + parse_metric(value), + parse_filter_key_string(value) + } do + {{:ok, time}, _, _} -> {:ok, time} + {_, {:ok, metric}, _} -> {:ok, metric} + {_, _, {:ok, dimension}} -> {:ok, dimension} + _ -> {:error, "Invalid order_by entry '#{inspect(entry)}'"} + end + end + + defp parse_time("time"), do: {:ok, "time"} + defp parse_time("time:hour"), do: {:ok, "time:hour"} + defp parse_time("time:day"), do: {:ok, "time:day"} + defp parse_time("time:month"), do: {:ok, "time:month"} + defp parse_time(_), do: :error + + defp parse_order_direction([_, "asc"]), do: {:ok, :asc} + defp parse_order_direction([_, "desc"]), do: {:ok, :desc} + defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"} + + defp parse_include(%{"imports" => value}) when is_boolean(value), do: {:ok, %{imports: value}} + defp parse_include(%{}), do: {:ok, %{}} + defp parse_include(include), do: {:error, "Invalid include passed '#{inspect(include)}'"} + + defp parse_filter_key_string(filter_key, error_message \\ "") do + case filter_key do + "event:props:" <> property_name -> + if String.length(property_name) > 0 do + {:ok, filter_key} + else + {:error, error_message} + end + + "event:" <> key -> + if key in Filters.event_props() do + {:ok, filter_key} + else + {:error, error_message} + end + + "visit:" <> key -> + if key in Filters.visit_props() do + {:ok, filter_key} + else + {:error, error_message} + end + + _ -> + {:error, error_message} + end + end + + defp validate_order_by(query) do + if query.order_by do + valid_values = query.metrics ++ query.dimensions + + invalid_entry = + Enum.find(query.order_by, fn {value, _direction} -> + not Enum.member?(valid_values, value) + end) + + case invalid_entry do + nil -> + :ok + + _ -> + {:error, + "Invalid order_by entry '#{inspect(invalid_entry)}'. Entry is not a queried metric or dimension"} + end + else + :ok + end + end + + def preload_goals_if_needed(site, filters, dimensions) do + goal_filters? = + Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end) + + if goal_filters? or Enum.member?(dimensions, "event:goal") do + Plausible.Goals.for_site(site) + |> Enum.map(fn + %{page_path: path} when is_binary(path) -> {:page, path} + %{event_name: event_name} -> {:event, event_name} + end) + else + [] + end + end + + defp validate_goal_filters(query) do + goal_filter_clauses = + Enum.flat_map(query.filters, fn + [_operation, "event:goal", clauses] -> clauses + _ -> [] + end) + + if length(goal_filter_clauses) > 0 do + validate_list(goal_filter_clauses, &validate_goal_filter(&1, query.preloaded_goals)) + else + :ok + end + end + + defp validate_goal_filter(clause, configured_goals) do + if Enum.member?(configured_goals, clause) do + :ok + else + {:error, + "The goal `#{Filters.Utils.unwrap_goal_value(clause)}` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"} + end + end + + defp validate_custom_props_access(site, query) do + allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true) + + validate_custom_props_access(site, query, allowed_props) + end + + defp validate_custom_props_access(_site, _query, :all), do: :ok + + defp validate_custom_props_access(_site, query, allowed_props) do + valid? = + query.filters + |> Enum.map(fn [_operation, filter_key | _rest] -> filter_key end) + |> Enum.concat(query.dimensions) + |> Enum.all?(fn + "event:props:" <> prop -> prop in allowed_props + _ -> true + end) + + if valid? do + :ok + else + {:error, "The owner of this site does not have access to the custom properties feature"} + end + end + + defp validate_metrics(query) do + with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do + validate_no_metrics_filters_conflict(query) + end + end + + defp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] do + if Enum.member?(query.dimensions, "event:goal") or + not is_nil(Query.get_filter(query, "event:goal")) do + :ok + else + {:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions"} + end + end + + defp validate_metric(:views_per_visit = metric, query) do + cond do + not is_nil(Query.get_filter(query, "event:page")) -> + {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`"} + + length(query.dimensions) > 0 -> + {:error, "Metric `#{metric}` cannot be queried with `dimensions`"} + + true -> + :ok + end + end + + defp validate_metric(_, _), do: :ok + + defp validate_no_metrics_filters_conflict(query) do + {_event_metrics, sessions_metrics, _other_metrics} = + TableDecider.partition_metrics(query.metrics, query) + + if Enum.empty?(sessions_metrics) or + not event_dimensions_not_allowing_session_metrics?(query.dimensions) do + :ok + else + {:error, + "Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event dimensions"} + end + end + + def event_dimensions_not_allowing_session_metrics?(dimensions) do + Enum.any?(dimensions, fn + "event:page" -> false + "event:" <> _ -> true + _ -> false + end) + end + + defp parse_list(list, parser_function) do + Enum.reduce_while(list, {:ok, []}, fn value, {:ok, results} -> + case parser_function.(value) do + {:ok, result} -> {:cont, {:ok, results ++ [result]}} + {:error, _} = error -> {:halt, error} + end + end) + end + + defp validate_list(list, parser_function) do + Enum.reduce_while(list, :ok, fn value, :ok -> + case parser_function.(value) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end +end diff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/stats_api_filter_parser.ex index fc58168993fa..5f02b6c8c633 100644 --- a/lib/plausible/stats/filters/stats_api_filter_parser.ex +++ b/lib/plausible/stats/filters/stats_api_filter_parser.ex @@ -27,11 +27,11 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do final_value = remove_escape_chars(raw_value) cond do - is_wildcard? && is_negated? -> [:does_not_match, key, raw_value] - is_wildcard? -> [:matches, key, raw_value] - is_list? -> [:member, key, parse_member_list(raw_value)] - is_negated? -> [:is_not, key, final_value] - true -> [:is, key, final_value] + is_wildcard? && is_negated? -> [:does_not_match, key, [raw_value]] + is_wildcard? -> [:matches, key, [raw_value]] + is_list? -> [:is, key, parse_member_list(raw_value)] + is_negated? -> [:is_not, key, [final_value]] + true -> [:is, key, [final_value]] end |> reject_invalid_country_codes() @@ -71,10 +71,10 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do |> wrap_goal_value() cond do - is_list? && is_wildcard? -> [:matches_member, key, value] - is_list? -> [:member, key, value] - is_wildcard? -> [:matches, key, value] - true -> [:is, key, value] + is_list? && is_wildcard? -> [:matches, key, value] + is_list? -> [:is, key, value] + is_wildcard? -> [:matches, key, [value]] + true -> [:is, key, [value]] end end end diff --git a/lib/plausible/stats/filters/utils.ex b/lib/plausible/stats/filters/utils.ex index 5bb612b62e3c..96606c1d02ef 100644 --- a/lib/plausible/stats/filters/utils.ex +++ b/lib/plausible/stats/filters/utils.ex @@ -1,6 +1,6 @@ defmodule Plausible.Stats.Filters.Utils do @moduledoc """ - Contains utility functions shared between `DashboardFilterParser` + Contains utility functions shared between `LegacyDashboardFilterParser` and `StatsAPIFilterParser`. """ @@ -66,4 +66,18 @@ defmodule Plausible.Stats.Filters.Utils do def unwrap_goal_value(goals) when is_list(goals), do: Enum.map(goals, &unwrap_goal_value/1) def unwrap_goal_value({:page, page}), do: "Visit " <> page def unwrap_goal_value({:event, event}), do: event + + def split_goals(goals) do + Enum.split_with(goals, fn {type, _} -> type == :event end) + end + + def split_goals_query_expressions(goals) do + {event_goals, pageview_goals} = split_goals(goals) + events = Enum.map(event_goals, fn {_, event} -> event end) + + page_regexes = + Enum.map(pageview_goals, fn {_, path} -> Plausible.Stats.Base.page_regex(path) end) + + {events, page_regexes} + end end diff --git a/lib/plausible/stats/filters/where_builder.ex b/lib/plausible/stats/filters/where_builder.ex index 04f66c836344..257d4257a027 100644 --- a/lib/plausible/stats/filters/where_builder.ex +++ b/lib/plausible/stats/filters/where_builder.ex @@ -10,6 +10,8 @@ defmodule Plausible.Stats.Filters.WhereBuilder do use Plausible.Stats.Fragments + require Logger + @sessions_only_visit_fields [ :entry_page, :exit_page, @@ -68,57 +70,38 @@ defmodule Plausible.Stats.Filters.WhereBuilder do ) end - defp add_filter(:events, _query, [:is, "event:name", name]) do - dynamic([e], e.name == ^name) - end - - defp add_filter(:events, _query, [:member, "event:name", list]) do + defp add_filter(:events, _query, [:is, "event:name", list]) do dynamic([e], e.name in ^list) end - defp add_filter(:events, _query, [:is, "event:goal", {:page, path}]) do - dynamic([e], e.pathname == ^path and e.name == "pageview") - end - - defp add_filter(:events, _query, [:matches, "event:goal", {:page, expr}]) do - regex = page_regex(expr) + defp add_filter(:events, _query, [operation, "event:goal", clauses]) + when operation in [:is, :matches] do + {events, pages, wildcard?} = split_goals(clauses) - dynamic([e], fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview") - end + if wildcard? do + event_clause = + if Enum.any?(events) do + dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events)) + else + dynamic([x], false) + end - defp add_filter(:events, _query, [:is, "event:goal", {:event, event}]) do - dynamic([e], e.name == ^event) - end + page_clause = + if Enum.any?(pages) do + dynamic( + [x], + fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview" + ) + else + dynamic([x], false) + end - defp add_filter(:events, _query, [:member, "event:goal", clauses]) do - {events, pages} = split_goals(clauses) + where_clause = dynamic([], ^event_clause or ^page_clause) - dynamic([e], (e.pathname in ^pages and e.name == "pageview") or e.name in ^events) - end - - defp add_filter(:events, _query, [:matches_member, "event:goal", clauses]) do - {events, pages} = split_goals(clauses, &page_regex/1) - - event_clause = - if Enum.any?(events) do - dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events)) - else - dynamic([x], false) - end - - page_clause = - if Enum.any?(pages) do - dynamic( - [x], - fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview" - ) - else - dynamic([x], false) - end - - where_clause = dynamic([], ^event_clause or ^page_clause) - - dynamic([e], ^where_clause) + dynamic([e], ^where_clause) + else + dynamic([e], (e.pathname in ^pages and e.name == "pageview") or e.name in ^events) + end end defp add_filter(:events, _query, [_, "event:page" | _rest] = filter) do @@ -169,39 +152,16 @@ defmodule Plausible.Stats.Filters.WhereBuilder do true end - defp filter_custom_prop(prop_name, column_name, [:is, _, "(none)"]) do - dynamic([t], not has_key(t, column_name, ^prop_name)) - end - - defp filter_custom_prop(prop_name, column_name, [:is, _, value]) do - dynamic( - [t], - has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) == ^value + defp add_filter(table, _query, filter) do + Logger.info("Unable to process garbage filter. No results are returned", + table: table, + filter: filter ) - end - defp filter_custom_prop(prop_name, column_name, [:is_not, _, "(none)"]) do - dynamic([t], has_key(t, column_name, ^prop_name)) - end - - defp filter_custom_prop(prop_name, column_name, [:is_not, _, value]) do - dynamic( - [t], - not has_key(t, column_name, ^prop_name) or get_by_key(t, column_name, ^prop_name) != ^value - ) + false end - defp filter_custom_prop(prop_name, column_name, [:matches, _, value]) do - regex = page_regex(value) - - dynamic( - [t], - has_key(t, column_name, ^prop_name) and - fragment("match(?, ?)", get_by_key(t, column_name, ^prop_name), ^regex) - ) - end - - defp filter_custom_prop(prop_name, column_name, [:member, _, values]) do + defp filter_custom_prop(prop_name, column_name, [:is, _, values]) do none_value_included = Enum.member?(values, "(none)") dynamic( @@ -211,7 +171,7 @@ defmodule Plausible.Stats.Filters.WhereBuilder do ) end - defp filter_custom_prop(prop_name, column_name, [:not_member, _, values]) do + defp filter_custom_prop(prop_name, column_name, [:is_not, _, values]) do none_value_included = Enum.member?(values, "(none)") dynamic( @@ -225,7 +185,7 @@ defmodule Plausible.Stats.Filters.WhereBuilder do ) end - defp filter_custom_prop(prop_name, column_name, [:matches_member, _, clauses]) do + defp filter_custom_prop(prop_name, column_name, [:matches, _, clauses]) do regexes = Enum.map(clauses, &page_regex/1) dynamic( @@ -239,42 +199,68 @@ defmodule Plausible.Stats.Filters.WhereBuilder do ) end - defp filter_field(db_field, [:is, _key, value]) do - value = db_field_val(db_field, value) - dynamic([x], field(x, ^db_field) == ^value) + defp filter_custom_prop(prop_name, column_name, [:does_not_match, _, clauses]) do + regexes = Enum.map(clauses, &page_regex/1) + + dynamic( + [t], + has_key(t, column_name, ^prop_name) and + fragment( + "not(arrayExists(k -> match(?, k), ?))", + get_by_key(t, column_name, ^prop_name), + ^regexes + ) + ) + end + + defp filter_custom_prop(prop_name, column_name, [:contains, _, clauses]) do + dynamic( + [t], + has_key(t, column_name, ^prop_name) and + fragment( + "multiSearchAny(?, ?)", + get_by_key(t, column_name, ^prop_name), + ^clauses + ) + ) end - defp filter_field(db_field, [:is_not, _key, value]) do - value = db_field_val(db_field, value) - dynamic([x], field(x, ^db_field) != ^value) + defp filter_custom_prop(prop_name, column_name, [:does_not_contain, _, clauses]) do + dynamic( + [t], + has_key(t, column_name, ^prop_name) and + fragment( + "not(multiSearchAny(?, ?))", + get_by_key(t, column_name, ^prop_name), + ^clauses + ) + ) end - defp filter_field(db_field, [:matches_member, _key, glob_exprs]) do + defp filter_field(db_field, [:matches, _key, glob_exprs]) do page_regexes = Enum.map(glob_exprs, &page_regex/1) dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes)) end - defp filter_field(db_field, [:not_matches_member, _key, glob_exprs]) do + defp filter_field(db_field, [:does_not_match, _key, glob_exprs]) do page_regexes = Enum.map(glob_exprs, &page_regex/1) dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes)) end - defp filter_field(db_field, [:matches, _key, glob_expr]) do - regex = page_regex(glob_expr) - dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex)) + defp filter_field(db_field, [:contains, _key, values]) do + dynamic([x], fragment("multiSearchAny(?, ?)", field(x, ^db_field), ^values)) end - defp filter_field(db_field, [:does_not_match, _key, glob_expr]) do - regex = page_regex(glob_expr) - dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex)) + defp filter_field(db_field, [:does_not_contain, _key, values]) do + dynamic([x], fragment("not(multiSearchAny(?, ?))", field(x, ^db_field), ^values)) end - defp filter_field(db_field, [:member, _key, list]) do + defp filter_field(db_field, [:is, _key, list]) do list = Enum.map(list, &db_field_val(db_field, &1)) dynamic([x], field(x, ^db_field) in ^list) end - defp filter_field(db_field, [:not_member, _key, list]) do + defp filter_field(db_field, [:is_not, _key, list]) do list = Enum.map(list, &db_field_val(db_field, &1)) dynamic([x], field(x, ^db_field) not in ^list) end @@ -292,13 +278,14 @@ defmodule Plausible.Stats.Filters.WhereBuilder do defp db_field_val(_, @not_set), do: "" defp db_field_val(_, val), do: val - defp split_goals(clauses, map_fn \\ &Function.identity/1) do - groups = - Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end) + defp split_goals(clauses) do + wildcard? = Enum.any?(clauses, fn {_, value} -> String.contains?(value, "*") end) + map_fn = if(wildcard?, do: &page_regex/1, else: &Function.identity/1) - { - Map.get(groups, :event, []), - Map.get(groups, :page, []) - } + clauses + |> Enum.reduce({[], [], wildcard?}, fn + {:event, value}, {event, page, wildcard?} -> {event ++ [map_fn.(value)], page, wildcard?} + {:page, value}, {event, page, wildcard?} -> {event, page ++ [map_fn.(value)], wildcard?} + end) end end diff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.ex index 7802478f5e7c..b55ee934dd8b 100644 --- a/lib/plausible/stats/goal_suggestions.ex +++ b/lib/plausible/stats/goal_suggestions.ex @@ -24,6 +24,7 @@ defmodule Plausible.Stats.GoalSuggestions do |> Plausible.Imported.load_import_data() excluded = Keyword.get(opts, :exclude, []) + limit = Keyword.get(opts, :limit, 25) params = %{"with_imported" => "true", "period" => "6mo"} query = Query.from(site, params) @@ -32,15 +33,17 @@ defmodule Plausible.Stats.GoalSuggestions do from(e in base_event_query(site, query), where: fragment("? ilike ?", e.name, ^matches), where: e.name != "pageview", + where: fragment("trim(?)", e.name) != "", + where: e.name == fragment("trim(?)", e.name), where: e.name not in ^excluded, select: %{ name: e.name, visitors: visitors(e) }, order_by: selected_as(:visitors), - group_by: e.name, - limit: 25 + group_by: e.name ) + |> maybe_set_limit(limit) imported_q = from(i in "imported_custom_events", @@ -49,24 +52,34 @@ defmodule Plausible.Stats.GoalSuggestions do where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last, where: i.visitors > 0, where: fragment("? ilike ?", i.name, ^matches), + where: fragment("trim(?)", i.name) != "", + where: i.name == fragment("trim(?)", i.name), where: i.name not in ^excluded, select: %{ name: i.name, visitors: selected_as(sum(i.visitors), :visitors) }, order_by: selected_as(:visitors), - group_by: i.name, - limit: 25 + group_by: i.name ) + |> maybe_set_limit(limit) from(e in Ecto.Query.subquery(native_q), full_join: i in subquery(imported_q), on: e.name == i.name, select: selected_as(fragment("if(empty(?), ?, ?)", e.name, i.name, e.name), :name), - order_by: [desc: e.visitors + i.visitors], - limit: 25 + order_by: [desc: e.visitors + i.visitors] ) + |> maybe_set_limit(limit) |> ClickhouseRepo.all() |> Enum.reject(&(String.length(&1) > Plausible.Goal.max_event_name_length())) end + + defp maybe_set_limit(q, :unlimited) do + q + end + + defp maybe_set_limit(q, limit) when is_integer(limit) and limit > 0 do + limit(q, ^limit) + end end diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex index 539f3fb83cac..5b774d9ea960 100644 --- a/lib/plausible/stats/imported/base.ex +++ b/lib/plausible/stats/imported/base.ex @@ -54,10 +54,12 @@ defmodule Plausible.Stats.Imported.Base do def property_to_table_mappings(), do: @property_to_table_mappings def query_imported(site, query) do - query - |> transform_filters() - |> decide_table() - |> query_imported(site, query) + [table] = + query + |> transform_filters() + |> decide_tables() + + query_imported(table, site, query) end def query_imported(table, site, query) do @@ -75,13 +77,13 @@ defmodule Plausible.Stats.Imported.Base do |> apply_filter(query) end - def decide_table(query) do + def decide_tables(query) do query = transform_filters(query) if custom_prop_query?(query) do do_decide_custom_prop_table(query) else - do_decide_table(query) + do_decide_tables(query) end end @@ -89,28 +91,20 @@ defmodule Plausible.Stats.Imported.Base do new_filters = query.filters |> Enum.reject(fn - [:is, "event:name", "pageview"] -> true + [:is, "event:name", ["pageview"]] -> true _ -> false end) - |> Enum.flat_map(fn filter -> - case filter do - [op, "event:goal", {:event, name}] -> - [[op, "event:name", name]] - - [op, "event:goal", {:page, page}] -> - [[op, "event:page", page]] - - [op, "event:goal", events] -> - events - |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) - |> Enum.map(fn - {:event, names} -> [op, "event:name", names] - {:page, pages} -> [op, "event:page", pages] - end) - - filter -> - [filter] - end + |> Enum.flat_map(fn + [op, "event:goal", clauses] -> + clauses + |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + |> Enum.map(fn + {:event, names} -> [op, "event:name", names] + {:page, pages} -> [op, "event:page", pages] + end) + + filter -> + [filter] end) struct!(query, filters: new_filters) @@ -119,64 +113,72 @@ defmodule Plausible.Stats.Imported.Base do defp custom_prop_query?(query) do query.filters |> Enum.map(&Enum.at(&1, 1)) - |> Enum.concat([query.property]) + |> Enum.concat(query.dimensions) |> Enum.any?(&(&1 in @imported_custom_props)) end - defp do_decide_custom_prop_table(%{property: property} = query) - when property in @imported_custom_props do - do_decide_custom_prop_table(query, property) + defp do_decide_custom_prop_table(%{dimensions: [dimension]} = query) + when dimension in @imported_custom_props do + do_decide_custom_prop_table(query, dimension) end - defp do_decide_custom_prop_table(%{property: property} = query) - when property in [nil, "event:goal", "event:name"] do - custom_prop_filters = - query.filters - |> Enum.map(&Enum.at(&1, 1)) - |> Enum.filter(&(&1 in @imported_custom_props)) - |> Enum.uniq() - - case custom_prop_filters do - [custom_prop_filter] -> - do_decide_custom_prop_table(query, custom_prop_filter) - - _ -> - nil + defp do_decide_custom_prop_table(%{dimensions: dimensions} = query) do + if dimensions == [] or + (length(dimensions) == 1 and hd(dimensions) in ["event:goal", "event:name"]) do + custom_prop_filters = + query.filters + |> Enum.map(&Enum.at(&1, 1)) + |> Enum.filter(&(&1 in @imported_custom_props)) + |> Enum.uniq() + + case custom_prop_filters do + [custom_prop_filter] -> + do_decide_custom_prop_table(query, custom_prop_filter) + + _ -> + [] + end + else + [] end end - defp do_decide_custom_prop_table(_query), do: nil - defp do_decide_custom_prop_table(query, property) do has_required_name_filter? = - Enum.any?(query.filters, fn - [:is, "event:name", name] -> name in special_goals_for(property) - _ -> false + query.filters + |> Enum.flat_map(fn + [:is, "event:name", names] -> names + _ -> [] end) + |> Enum.any?(&(&1 in special_goals_for(property))) has_unsupported_filters? = - Enum.any?(query.filters, fn [_, filtered_prop | _] -> - filtered_prop not in [property, "event:name"] + Enum.any?(query.filters, fn [_, filter_key | _] -> + filter_key not in [property, "event:name"] end) if has_required_name_filter? and not has_unsupported_filters? do - "imported_custom_events" + ["imported_custom_events"] else - nil + [] end end - defp do_decide_table(%Query{filters: [], property: nil}), do: "imported_visitors" + defp do_decide_tables(%Query{filters: [], dimensions: []}), do: ["imported_visitors"] - defp do_decide_table(%Query{filters: [], property: "event:goal"}) do - "imported_custom_events" + defp do_decide_tables(%Query{filters: [], dimensions: ["event:goal"]}) do + ["imported_pages", "imported_custom_events"] end - defp do_decide_table(%Query{filters: [], property: property}) do - @property_to_table_mappings[property] + defp do_decide_tables(%Query{filters: [], dimensions: [dimension]}) do + if Map.has_key?(@property_to_table_mappings, dimension) do + [@property_to_table_mappings[dimension]] + else + [] + end end - defp do_decide_table(%Query{filters: filters, property: "event:goal"}) do + defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]}) do filter_props = Enum.map(filters, &Enum.at(&1, 1)) any_event_name_filters? = "event:name" in filter_props @@ -184,33 +186,34 @@ defmodule Plausible.Stats.Imported.Base do any_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"])) cond do - any_other_filters? -> nil - any_event_name_filters? and not any_page_filters? -> "imported_custom_events" - any_page_filters? and not any_event_name_filters? -> "imported_pages" - true -> nil + any_other_filters? -> [] + any_event_name_filters? and not any_page_filters? -> ["imported_custom_events"] + any_page_filters? and not any_event_name_filters? -> ["imported_pages"] + true -> [] end end - defp do_decide_table(%Query{filters: filters, property: property}) do + defp do_decide_tables(%Query{filters: filters, dimensions: dimensions}) do table_candidates = filters - |> Enum.map(fn [_, prop | _] -> prop end) - |> Enum.concat(if property, do: [property], else: []) + |> Enum.map(fn [_, filter_key | _] -> filter_key end) + |> Enum.concat(dimensions) |> Enum.map(fn "visit:screen" -> "visit:device" - prop -> prop + dimension -> dimension end) |> Enum.map(&@property_to_table_mappings[&1]) case Enum.uniq(table_candidates) do - [candidate] -> candidate - _ -> nil + [nil] -> [] + [candidate] -> [candidate] + _ -> [] end end defp apply_filter(q, %Query{filters: filters}) do - Enum.reduce(filters, q, fn [_, filtered_prop | _] = filter, q -> - db_field = Plausible.Stats.Filters.without_prefix(filtered_prop) + Enum.reduce(filters, q, fn [_, filter_key | _] = filter, q -> + db_field = Plausible.Stats.Filters.without_prefix(filter_key) mapped_db_field = Map.get(@db_field_mappings, db_field, db_field) condition = Filters.WhereBuilder.build_condition(mapped_db_field, filter) diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex index a387ab1a24f5..57cd6737bb19 100644 --- a/lib/plausible/stats/imported/imported.ex +++ b/lib/plausible/stats/imported/imported.ex @@ -1,12 +1,14 @@ defmodule Plausible.Stats.Imported do + alias Plausible.Stats.Filters use Plausible.ClickhouseRepo import Ecto.Query import Plausible.Stats.Fragments + import Plausible.Stats.Util, only: [shortname: 2] - alias Plausible.Stats.Base alias Plausible.Stats.Imported alias Plausible.Stats.Query + alias Plausible.Stats.SQL.QueryBuilder @no_ref "Direct / None" @not_set "(not set)" @@ -14,7 +16,7 @@ defmodule Plausible.Stats.Imported do @property_to_table_mappings Imported.Base.property_to_table_mappings() - @imported_properties Map.keys(@property_to_table_mappings) ++ + @imported_dimensions Map.keys(@property_to_table_mappings) ++ Plausible.Imported.imported_custom_props() @goals_with_url Plausible.Imported.goals_with_url() @@ -36,7 +38,7 @@ defmodule Plausible.Stats.Imported do (see `@goals_with_url` and `@goals_with_path`). """ def schema_supports_query?(query) do - not is_nil(Imported.Base.decide_table(query)) + length(Imported.Base.decide_tables(query)) > 0 end def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{ @@ -266,43 +268,7 @@ defmodule Plausible.Stats.Imported do def merge_imported(q, _, %Query{include_imported: false}, _), do: q - def merge_imported(q, site, %Query{property: property} = query, metrics) - when property in @imported_properties do - dim = Plausible.Stats.Filters.without_prefix(property) - - imported_q = - site - |> Imported.Base.query_imported(query) - |> where([i], i.visitors > 0) - |> group_imported_by(dim) - |> select_imported_metrics(metrics) - - join_on = - case dim do - _ when dim in [:url, :path] -> - dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value) - - :os_version -> - dynamic([s, i], s.os == i.os and s.os_version == i.os_version) - - :browser_version -> - dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version) - - dim -> - dynamic([s, i], field(s, ^dim) == field(i, ^dim)) - end - - from(s in Ecto.Query.subquery(q), - full_join: i in subquery(imported_q), - on: ^join_on, - select: %{} - ) - |> select_joined_dimension(dim) - |> select_joined_metrics(metrics) - |> apply_order_by(metrics) - end - - def merge_imported(q, site, %Query{property: nil} = query, metrics) do + def merge_imported(q, site, %Query{dimensions: []} = query, metrics) do imported_q = site |> Imported.Base.query_imported(query) @@ -316,41 +282,76 @@ defmodule Plausible.Stats.Imported do |> select_joined_metrics(metrics) end - def merge_imported(q, _, _, _), do: q + def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) do + {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals) - def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q - - def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do - if Imported.Base.decide_table(query) == "imported_pages" do - page_regexes = Enum.map(page_exprs, &Base.page_regex/1) + Imported.Base.decide_tables(query) + |> Enum.map(fn + "imported_custom_events" -> + Imported.Base.query_imported("imported_custom_events", site, query) + |> where([i], i.visitors > 0) + |> select_merge([i], %{ + dim0: + selected_as( + fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name), + :dim0 + ) + }) + |> select_imported_metrics(metrics) + |> group_by([], selected_as(:dim0)) + |> where([], selected_as(:dim0) != 0) - imported_q = - "imported_pages" - |> Imported.Base.query_imported(site, query) + "imported_pages" -> + Imported.Base.query_imported("imported_pages", site, query) |> where([i], i.visitors > 0) |> where( [i], - fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes) + fragment( + "notEmpty(multiMatchAllIndices(?, ?) as indices)", + i.page, + type(^page_regexes, {:array, :string}) + ) ) |> join(:array, index in fragment("indices")) |> group_by([_i, index], index) |> select_merge([_i, index], %{ - name: fragment("concat('Visit ', ?[?])", ^page_exprs, index) + dim0: selected_as(type(fragment("?", index), :integer), :dim0) }) |> select_imported_metrics(metrics) + end) + |> Enum.reduce(q, fn imports_q, q -> + naive_dimension_join(q, imports_q, metrics) + end) + end + + def merge_imported(q, site, %Query{dimensions: dimensions} = query, metrics) do + if merge_imported_dimensions?(dimensions) do + imported_q = + site + |> Imported.Base.query_imported(query) + |> where([i], i.visitors > 0) + |> group_imported_by(query) + |> select_imported_metrics(metrics) - from(s in Ecto.Query.subquery(q), + from(s in subquery(q), full_join: i in subquery(imported_q), - on: s.name == i.name, + on: ^QueryBuilder.build_group_by_join(query), select: %{} ) - |> select_joined_dimension(:name) + |> select_joined_dimensions(query) |> select_joined_metrics(metrics) else q end end + def merge_imported(q, _, _, _), do: q + + defp merge_imported_dimensions?(dimensions) do + dimensions in [["visit:browser", "visit:browser_version"], ["visit:os", "visit:os_version"]] or + (length(dimensions) == 1 and hd(dimensions) in @imported_dimensions) + end + def total_imported_visitors(site, query) do site |> Imported.Base.query_imported(query) @@ -551,162 +552,179 @@ defmodule Plausible.Stats.Imported do |> select_imported_metrics(rest) end - defp group_imported_by(q, dim) when dim in [:source, :referrer] do + defp group_imported_by(q, query) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + dim = Plausible.Stats.Filters.without_prefix(dimension) + + group_imported_by(q, dim, shortname(query, dimension)) + end) + end + + defp group_imported_by(q, dim, key) when dim in [:source, :referrer] do q |> group_by([i], field(i, ^dim)) |> select_merge([i], %{ - ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim)) + ^key => + selected_as( + fragment( + "if(empty(?), ?, ?)", + field(i, ^dim), + @no_ref, + field(i, ^dim) + ), + ^key + ) }) end - defp group_imported_by(q, dim) + defp group_imported_by(q, dim, key) when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do q |> group_by([i], field(i, ^dim)) |> where([i], fragment("not empty(?)", field(i, ^dim))) - |> select_merge([i], %{^dim => field(i, ^dim)}) + |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)}) end - defp group_imported_by(q, :page) do + defp group_imported_by(q, :page, key) do q |> group_by([i], i.page) - |> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)}) + |> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)}) end - defp group_imported_by(q, :country) do + defp group_imported_by(q, :country, key) do q |> group_by([i], i.country) |> where([i], i.country != "ZZ") - |> select_merge([i], %{country: i.country}) + |> select_merge([i], %{^key => selected_as(i.country, ^key)}) end - defp group_imported_by(q, :region) do + defp group_imported_by(q, :region, key) do q |> group_by([i], i.region) |> where([i], i.region != "") - |> select_merge([i], %{region: i.region}) + |> select_merge([i], %{^key => selected_as(i.region, ^key)}) end - defp group_imported_by(q, :city) do + defp group_imported_by(q, :city, key) do q |> group_by([i], i.city) |> where([i], i.city != 0 and not is_nil(i.city)) - |> select_merge([i], %{city: i.city}) + |> select_merge([i], %{^key => selected_as(i.city, ^key)}) end - defp group_imported_by(q, dim) when dim in [:device, :browser] do + defp group_imported_by(q, dim, key) when dim in [:device, :browser] do q |> group_by([i], field(i, ^dim)) |> select_merge([i], %{ - ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)) + ^key => + selected_as( + fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)), + ^key + ) }) end - defp group_imported_by(q, :browser_version) do + defp group_imported_by(q, :browser_version, key) do q - |> group_by([i], [i.browser, i.browser_version]) + |> group_by([i], [i.browser_version]) |> select_merge([i], %{ - browser: fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser), - browser_version: - fragment( - "if(empty(?), ?, ?)", - i.browser_version, - @not_set, - i.browser_version + ^key => + selected_as( + fragment( + "if(empty(?), ?, ?)", + i.browser_version, + @not_set, + i.browser_version + ), + ^key ) }) end - defp group_imported_by(q, :os) do + defp group_imported_by(q, :os, key) do q |> group_by([i], i.operating_system) |> select_merge([i], %{ - os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system) + ^key => + selected_as( + fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system), + ^key + ) }) end - defp group_imported_by(q, :os_version) do + defp group_imported_by(q, :os_version, key) do q - |> group_by([i], [i.operating_system, i.operating_system_version]) + |> group_by([i], [i.operating_system_version]) |> select_merge([i], %{ - os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system), - os_version: - fragment( - "if(empty(?), ?, ?)", - i.operating_system_version, - @not_set, - i.operating_system_version + ^key => + selected_as( + fragment( + "if(empty(?), ?, ?)", + i.operating_system_version, + @not_set, + i.operating_system_version + ), + ^key ) }) end - defp group_imported_by(q, dim) when dim in [:entry_page, :exit_page] do + defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] do q |> group_by([i], field(i, ^dim)) - |> select_merge([i], %{^dim => field(i, ^dim)}) + |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)}) end - defp group_imported_by(q, :name) do + defp group_imported_by(q, :name, key) do q |> group_by([i], i.name) - |> select_merge([i], %{name: i.name}) + |> select_merge([i], %{^key => selected_as(i.name, ^key)}) end - defp group_imported_by(q, :url) do + defp group_imported_by(q, :url, key) do q |> group_by([i], i.link_url) |> select_merge([i], %{ - breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) + ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key) }) end - defp group_imported_by(q, :path) do + defp group_imported_by(q, :path, key) do q |> group_by([i], i.path) |> select_merge([i], %{ - breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) + ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key) }) end - defp select_joined_dimension(q, :city) do - select_merge(q, [s, i], %{ - city: fragment("greatest(?,?)", i.city, s.city) - }) + defp select_joined_dimensions(q, query) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + select_joined_dimension(q, dimension, shortname(query, dimension)) + end) end - defp select_joined_dimension(q, :os_version) do + defp select_joined_dimension(q, "visit:city", key) do select_merge(q, [s, i], %{ - os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os), - os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version) + ^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key) }) end - defp select_joined_dimension(q, :browser_version) do + defp select_joined_dimension(q, _dimension, key) do select_merge(q, [s, i], %{ - browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser), - browser_version: - fragment("if(empty(?), ?, ?)", s.browser_version, i.browser_version, s.browser_version) - }) - end - - defp select_joined_dimension(q, dim) when dim in [:url, :path] do - select_merge(q, [s, i], %{ - breakdown_prop_value: - fragment( - "if(empty(?), ?, ?)", - s.breakdown_prop_value, - i.breakdown_prop_value, - s.breakdown_prop_value + ^key => + selected_as( + fragment( + "if(empty(?), ?, ?)", + field(s, ^key), + field(i, ^key), + field(s, ^key) + ), + ^key ) }) end - defp select_joined_dimension(q, dim) do - select_merge(q, [s, i], %{ - ^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim)) - }) - end - defp select_joined_metrics(q, []), do: q # NOTE: Reverse-engineering the native data bounces and total visit # durations to combine with imported data is inefficient. Instead both @@ -716,7 +734,7 @@ defmodule Plausible.Stats.Imported do defp select_joined_metrics(q, [:visits | rest]) do q - |> select_merge([s, i], %{visits: s.visits + i.visits}) + |> select_merge([s, i], %{visits: selected_as(s.visits + i.visits, :visits)}) |> select_joined_metrics(rest) end @@ -728,13 +746,13 @@ defmodule Plausible.Stats.Imported do defp select_joined_metrics(q, [:events | rest]) do q - |> select_merge([s, i], %{events: s.events + i.events}) + |> select_merge([s, i], %{events: selected_as(s.events + i.events, :events)}) |> select_joined_metrics(rest) end defp select_joined_metrics(q, [:pageviews | rest]) do q - |> select_merge([s, i], %{pageviews: s.pageviews + i.pageviews}) + |> select_merge([s, i], %{pageviews: selected_as(s.pageviews + i.pageviews, :pageviews)}) |> select_joined_metrics(rest) end @@ -809,10 +827,14 @@ defmodule Plausible.Stats.Imported do |> select_joined_metrics(rest) end - defp apply_order_by(q, [:visitors | rest]) do - order_by(q, [s, i], desc: s.visitors + i.visitors) - |> apply_order_by(rest) + defp naive_dimension_join(q1, q2, metrics) do + from(a in subquery(q1), + full_join: b in subquery(q2), + on: a.dim0 == b.dim0, + select: %{ + dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0) + } + ) + |> select_joined_metrics(metrics) end - - defp apply_order_by(q, _), do: q end diff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.ex index 36a1c3743e9a..29319d07a422 100644 --- a/lib/plausible/stats/metrics.ex +++ b/lib/plausible/stats/metrics.ex @@ -16,12 +16,20 @@ defmodule Plausible.Stats.Metrics do :visit_duration, :events, :conversion_rate, - :time_on_page + :group_conversion_rate, + :time_on_page, + :percentage ] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) @metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end) + def metric?(value), do: Enum.member?(@all_metrics, value) + def from_string!(str) do Map.fetch!(@metric_mappings, str) end + + def from_string(str) do + Map.fetch(@metric_mappings, str) + end end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 76041690222d..75fb8c503c08 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Query do defstruct date_range: nil, interval: nil, period: nil, - property: nil, + dimensions: [], filters: [], sample_threshold: 20_000_000, imported_data_requested: false, @@ -13,7 +13,17 @@ defmodule Plausible.Stats.Query do now: nil, experimental_session_count?: false, experimental_reduced_joins?: false, - latest_import_end_date: nil + latest_import_end_date: nil, + metrics: [], + order_by: nil, + timezone: nil, + v2: false, + legacy_breakdown: false, + preloaded_goals: [], + include: %{ + imports: false, + time_labels: false + } require OpenTelemetry.Tracer, as: Tracer alias Plausible.Stats.{Filters, Interval, Imported} @@ -30,7 +40,7 @@ defmodule Plausible.Stats.Query do |> put_experimental_session_count(site, params) |> put_experimental_reduced_joins(site, params) |> put_period(site, params) - |> put_breakdown_property(params) + |> put_dimensions(params) |> put_interval(params) |> put_parsed_filters(params) |> put_imported_opts(site, params) @@ -42,6 +52,19 @@ defmodule Plausible.Stats.Query do query end + def build(site, params) do + with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do + query = + struct!(__MODULE__, Map.to_list(query_data)) + |> put_imported_opts(site, %{}) + |> put_experimental_session_count(site, params) + |> put_experimental_reduced_joins(site, params) + |> struct!(v2: true) + + {:ok, query} + end + end + defp put_experimental_session_count(query, site, params) do if Map.has_key?(params, "experimental_session_count") do struct!(query, @@ -185,8 +208,12 @@ defmodule Plausible.Stats.Query do put_period(query, site, Map.merge(params, %{"period" => "30d"})) end - defp put_breakdown_property(query, params) do - struct!(query, property: params["property"]) + defp put_dimensions(query, params) do + if not is_nil(params["property"]) do + struct!(query, dimensions: [params["property"]]) + else + struct!(query, dimensions: Map.get(params, "dimensions", [])) + end end defp put_interval(%{:period => "all"} = query, params) do @@ -203,10 +230,28 @@ defmodule Plausible.Stats.Query do struct!(query, filters: Filters.parse(params["filters"])) end - @spec set_property(t(), String.t() | nil) :: t() - def set_property(query, property) do + def set(query, keywords) do + query + |> struct!(keywords) + |> refresh_imported_opts() + end + + @spec set_dimensions(t(), list(String.t())) :: t() + def set_dimensions(query, dimensions) do + query + |> struct!(dimensions: dimensions) + |> refresh_imported_opts() + end + + def set_metrics(query, metrics) do + query + |> struct!(metrics: metrics) + |> refresh_imported_opts() + end + + def set_order_by(query, order_by) do query - |> struct!(property: property) + |> struct!(order_by: order_by) |> refresh_imported_opts() end @@ -322,7 +367,7 @@ defmodule Plausible.Stats.Query do Tracer.set_attributes([ {"plausible.query.interval", query.interval}, {"plausible.query.period", query.period}, - {"plausible.query.breakdown_property", query.property}, + {"plausible.query.dimensions", query.dimensions |> Enum.join(";")}, {"plausible.query.include_imported", query.include_imported}, {"plausible.query.filter_keys", filter_keys}, {"plausible.query.metrics", metrics} diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex new file mode 100644 index 000000000000..25146d950d73 --- /dev/null +++ b/lib/plausible/stats/query_optimizer.ex @@ -0,0 +1,165 @@ +defmodule Plausible.Stats.QueryOptimizer do + @moduledoc """ + Methods to manipulate Query for business logic reasons before building an ecto query. + """ + + use Plausible + alias Plausible.Stats.{Query, TableDecider, Util} + + @doc """ + This module manipulates an existing query, updating it according to business logic. + + For example, it: + 1. Figures out what the right granularity to group by time is + 2. Adds a missing order_by clause to a query + 3. Updating "time" dimension in order_by to the right granularity + + """ + def optimize(query) do + Enum.reduce(pipeline(), query, fn step, acc -> step.(acc) end) + end + + @doc """ + Splits a query into event and sessions subcomponents as not all metrics can be + queried from a single table. + + event:page dimension is treated in a special way, doing a breakdown of visit:entry_page + for sessions. + """ + def split(query) do + {event_metrics, sessions_metrics, _other_metrics} = + query.metrics + |> Util.maybe_add_visitors_metric() + |> TableDecider.partition_metrics(query) + + { + Query.set_metrics(query, event_metrics), + split_sessions_query(query, sessions_metrics) + } + end + + defp pipeline() do + [ + &update_group_by_time/1, + &add_missing_order_by/1, + &update_time_in_order_by/1, + &extend_hostname_filters_to_visit/1 + ] + end + + defp add_missing_order_by(%Query{order_by: nil} = query) do + order_by = + case time_dimension(query) do + nil -> [{hd(query.metrics), :desc}] + time_dimension -> [{time_dimension, :asc}, {hd(query.metrics), :desc}] + end + + %Query{query | order_by: order_by} + end + + defp add_missing_order_by(query), do: query + + defp update_group_by_time( + %Query{ + date_range: %Date.Range{first: first, last: last} + } = query + ) do + dimensions = + query.dimensions + |> Enum.map(fn + "time" -> resolve_time_dimension(first, last) + entry -> entry + end) + + %Query{query | dimensions: dimensions} + end + + defp update_group_by_time(query), do: query + + defp resolve_time_dimension(first, last) do + cond do + Timex.diff(last, first, :hours) <= 48 -> "time:hour" + Timex.diff(last, first, :days) <= 40 -> "time:day" + true -> "time:month" + end + end + + defp update_time_in_order_by(query) do + order_by = + query.order_by + |> Enum.map(fn + {"time", direction} -> {time_dimension(query), direction} + entry -> entry + end) + + %Query{query | order_by: order_by} + end + + @dimensions_hostname_map %{ + "visit:source" => "visit:entry_page_hostname", + "visit:entry_page" => "visit:entry_page_hostname", + "visit:utm_medium" => "visit:entry_page_hostname", + "visit:utm_source" => "visit:entry_page_hostname", + "visit:utm_campaign" => "visit:entry_page_hostname", + "visit:utm_content" => "visit:entry_page_hostname", + "visit:utm_term" => "visit:entry_page_hostname", + "visit:referrer" => "visit:entry_page_hostname", + "visit:exit_page" => "visit:exit_page_hostname" + } + + # To avoid showing referrers across hostnames when event:hostname + # filter is present for breakdowns, add entry/exit page hostname + # filters + defp extend_hostname_filters_to_visit(query) do + hostname_filters = + query.filters + |> Enum.filter(fn [_operation, filter_key | _rest] -> filter_key == "event:hostname" end) + + if length(hostname_filters) > 0 do + extra_filters = + query.dimensions + |> Enum.flat_map(&hostname_filters_for_dimension(&1, hostname_filters)) + + %Query{query | filters: query.filters ++ extra_filters} + else + query + end + end + + defp hostname_filters_for_dimension(dimension, hostname_filters) do + if Map.has_key?(@dimensions_hostname_map, dimension) do + filter_key = Map.get(@dimensions_hostname_map, dimension) + + hostname_filters + |> Enum.map(fn [operation, _filter_key | rest] -> [operation, filter_key | rest] end) + else + [] + end + end + + defp time_dimension(query) do + Enum.find(query.dimensions, &String.starts_with?(&1, "time")) + end + + defp split_sessions_query(query, session_metrics) do + dimensions = + query.dimensions + |> Enum.map(fn + "event:page" -> "visit:entry_page" + dimension -> dimension + end) + + filters = + if "event:page" in query.dimensions do + query.filters + |> Enum.map(fn + [op, "event:page" | rest] -> [op, "visit:entry_page" | rest] + filter -> filter + end) + else + query.filters + end + + Query.set(query, filters: filters, metrics: session_metrics, dimensions: dimensions) + end +end diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex new file mode 100644 index 000000000000..b9f374444834 --- /dev/null +++ b/lib/plausible/stats/query_result.ex @@ -0,0 +1,68 @@ +defmodule Plausible.Stats.QueryResult do + @moduledoc false + + alias Plausible.Stats.Util + alias Plausible.Stats.Filters + alias Plausible.Stats.Query + + @derive Jason.Encoder + defstruct results: [], + query: nil, + meta: %{} + + def from(results, query) do + results_list = + results + |> Enum.map(fn entry -> + %{ + dimensions: Enum.map(query.dimensions, &dimension_label(&1, entry, query)), + metrics: Enum.map(query.metrics, &Map.get(entry, &1)) + } + end) + + struct!( + __MODULE__, + results: results_list, + query: %{ + metrics: query.metrics, + date_range: [query.date_range.first, query.date_range.last], + filters: query.filters |> Enum.map(&serializable_filter/1), + dimensions: query.dimensions, + order_by: query.order_by |> Enum.map(&Tuple.to_list/1) + }, + meta: meta(query) + ) + end + + defp meta(%Query{skip_imported_reason: :unsupported_query}) do + %{ + warning: + "Imported stats are not included in the results because query parameters are not supported. " <> + "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats" + } + end + + defp meta(_), do: %{} + + defp dimension_label("event:goal", entry, query) do + {events, paths} = Filters.Utils.split_goals(query.preloaded_goals) + + goal_index = Map.get(entry, Util.shortname(query, "event:goal")) + + # Closely coupled logic with Plausible.Stats.SQL.Expression.event_goal_join/2 + cond do + goal_index < 0 -> Enum.at(events, -goal_index - 1) |> Filters.Utils.unwrap_goal_value() + goal_index > 0 -> Enum.at(paths, goal_index - 1) |> Filters.Utils.unwrap_goal_value() + end + end + + defp dimension_label(dimension, entry, query) do + Map.get(entry, Util.shortname(query, dimension)) + end + + defp serializable_filter([operation, "event:goal", clauses]) do + [operation, "event:goal", Enum.map(clauses, &Filters.Utils.unwrap_goal_value/1)] + end + + defp serializable_filter(filter), do: filter +end diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex new file mode 100644 index 000000000000..d3f38a142815 --- /dev/null +++ b/lib/plausible/stats/sql/expression.ex @@ -0,0 +1,146 @@ +defmodule Plausible.Stats.SQL.Expression do + @moduledoc """ + This module is responsible for generating SQL/Ecto expressions + for dimensions used in query select, group_by and order_by. + """ + + import Ecto.Query + + use Plausible.Stats.Fragments + + @no_ref "Direct / None" + @not_set "(not set)" + + defmacrop field_or_blank_value(expr, empty_value, select_alias) do + quote do + dynamic( + [t], + selected_as( + fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)), + ^unquote(select_alias) + ) + ) + end + end + + def dimension("time:hour", query, select_alias) do + dynamic( + [t], + selected_as( + fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone), + ^select_alias + ) + ) + end + + def dimension("time:day", query, select_alias) do + dynamic( + [t], + selected_as( + fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone), + ^select_alias + ) + ) + end + + def dimension("time:month", query, select_alias) do + dynamic( + [t], + selected_as( + fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone), + ^select_alias + ) + ) + end + + def dimension("event:name", _query, select_alias), + do: dynamic([t], selected_as(t.name, ^select_alias)) + + def dimension("event:page", _query, select_alias), + do: dynamic([t], selected_as(t.pathname, ^select_alias)) + + def dimension("event:hostname", _query, select_alias), + do: dynamic([t], selected_as(t.hostname, ^select_alias)) + + def dimension("event:props:" <> property_name, _query, select_alias) do + dynamic( + [t], + selected_as( + fragment( + "if(not empty(?), ?, '(none)')", + get_by_key(t, :meta, ^property_name), + get_by_key(t, :meta, ^property_name) + ), + ^select_alias + ) + ) + end + + def dimension("visit:entry_page", _query, select_alias), + do: dynamic([t], selected_as(t.entry_page, ^select_alias)) + + def dimension("visit:exit_page", _query, select_alias), + do: dynamic([t], selected_as(t.exit_page, ^select_alias)) + + def dimension("visit:utm_medium", _query, select_alias), + do: field_or_blank_value(t.utm_medium, @not_set, select_alias) + + def dimension("visit:utm_source", _query, select_alias), + do: field_or_blank_value(t.utm_source, @not_set, select_alias) + + def dimension("visit:utm_campaign", _query, select_alias), + do: field_or_blank_value(t.utm_campaign, @not_set, select_alias) + + def dimension("visit:utm_content", _query, select_alias), + do: field_or_blank_value(t.utm_content, @not_set, select_alias) + + def dimension("visit:utm_term", _query, select_alias), + do: field_or_blank_value(t.utm_term, @not_set, select_alias) + + def dimension("visit:source", _query, select_alias), + do: field_or_blank_value(t.source, @no_ref, select_alias) + + def dimension("visit:referrer", _query, select_alias), + do: field_or_blank_value(t.referrer, @no_ref, select_alias) + + def dimension("visit:device", _query, select_alias), + do: field_or_blank_value(t.device, @not_set, select_alias) + + def dimension("visit:os", _query, select_alias), + do: field_or_blank_value(t.os, @not_set, select_alias) + + def dimension("visit:os_version", _query, select_alias), + do: field_or_blank_value(t.os_version, @not_set, select_alias) + + def dimension("visit:browser", _query, select_alias), + do: field_or_blank_value(t.browser, @not_set, select_alias) + + def dimension("visit:browser_version", _query, select_alias), + do: field_or_blank_value(t.browser_version, @not_set, select_alias) + + def dimension("visit:country", _query, select_alias), + do: dynamic([t], selected_as(t.country, ^select_alias)) + + def dimension("visit:region", _query, select_alias), + do: dynamic([t], selected_as(t.region, ^select_alias)) + + def dimension("visit:city", _query, select_alias), + do: dynamic([t], selected_as(t.city, ^select_alias)) + + defmacro event_goal_join(events, page_regexes) do + quote do + fragment( + """ + arrayPushFront( + CAST(multiMatchAllIndices(?, ?) AS Array(Int64)), + -indexOf(?, ?) + ) + """, + e.pathname, + type(^unquote(page_regexes), {:array, :string}), + type(^unquote(events), {:array, :string}), + e.name + ) + end + end +end diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex new file mode 100644 index 000000000000..7e5e49a887f1 --- /dev/null +++ b/lib/plausible/stats/sql/query_builder.ex @@ -0,0 +1,280 @@ +defmodule Plausible.Stats.SQL.QueryBuilder do + @moduledoc false + + use Plausible + + import Ecto.Query + import Plausible.Stats.Imported + import Plausible.Stats.Util + + alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters} + alias Plausible.Stats.SQL.Expression + + require Plausible.Stats.SQL.Expression + + def build(query, site) do + {event_query, sessions_query} = QueryOptimizer.split(query) + + event_q = build_events_query(site, event_query) + sessions_q = build_sessions_query(site, sessions_query) + + join_query_results( + {event_q, event_query}, + {sessions_q, sessions_query} + ) + end + + defp build_events_query(_site, %Query{metrics: []}), do: nil + + defp build_events_query(site, events_query) do + q = + from( + e in "events_v2", + where: ^Filters.WhereBuilder.build(:events, site, events_query), + select: ^Base.select_event_metrics(events_query.metrics) + ) + + on_ee do + q = Plausible.Stats.Sampling.add_query_hint(q, events_query) + end + + q + |> join_sessions_if_needed(site, events_query) + |> build_group_by(events_query) + |> merge_imported(site, events_query, events_query.metrics) + |> maybe_add_global_conversion_rate(site, events_query) + |> maybe_add_group_conversion_rate(site, events_query) + |> Base.add_percentage_metric(site, events_query, events_query.metrics) + end + + defp join_sessions_if_needed(q, site, query) do + if TableDecider.events_join_sessions?(query) do + sessions_q = + from( + s in Base.query_sessions(site, query), + select: %{session_id: s.session_id}, + where: s.sign == 1, + group_by: s.session_id + ) + + from( + e in q, + join: sq in subquery(sessions_q), + on: e.session_id == sq.session_id + ) + else + q + end + end + + defp build_sessions_query(_site, %Query{metrics: []}), do: nil + + defp build_sessions_query(site, sessions_query) do + q = + from( + e in "sessions_v2", + where: ^Filters.WhereBuilder.build(:sessions, site, sessions_query), + select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query) + ) + + on_ee do + q = Plausible.Stats.Sampling.add_query_hint(q, sessions_query) + end + + q + |> join_events_if_needed(site, sessions_query) + |> build_group_by(sessions_query) + |> merge_imported(site, sessions_query, sessions_query.metrics) + |> maybe_add_global_conversion_rate(site, sessions_query) + |> maybe_add_group_conversion_rate(site, sessions_query) + |> Base.add_percentage_metric(site, sessions_query, sessions_query.metrics) + end + + def join_events_if_needed(q, site, query) do + if Query.has_event_filters?(query) do + events_q = + from(e in "events_v2", + where: ^Filters.WhereBuilder.build(:events, site, query), + select: %{ + session_id: fragment("DISTINCT ?", e.session_id), + _sample_factor: fragment("_sample_factor") + } + ) + + on_ee do + events_q = Plausible.Stats.Sampling.add_query_hint(events_q, query) + end + + from(s in q, + join: e in subquery(events_q), + on: s.session_id == e.session_id + ) + else + q + end + end + + defp build_group_by(q, query) do + Enum.reduce(query.dimensions, q, &dimension_group_by(&2, query, &1)) + end + + defp dimension_group_by(q, query, "event:goal" = dimension) do + {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals) + + from(e in q, + array_join: goal in Expression.event_goal_join(events, page_regexes), + select_merge: %{ + ^shortname(query, dimension) => fragment("?", goal) + }, + group_by: goal, + where: goal != 0 and (e.name == "pageview" or goal < 0) + ) + end + + defp dimension_group_by(q, query, dimension) do + key = shortname(query, dimension) + + q + |> select_merge(^%{key => Expression.dimension(dimension, query, key)}) + |> group_by([], selected_as(^key)) + end + + defp build_order_by(q, query) do + Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1)) + end + + def build_order_by(q, query, {metric_or_dimension, order_direction}) do + order_by( + q, + [t], + ^{ + order_direction, + dynamic([], selected_as(^shortname(query, metric_or_dimension))) + } + ) + end + + defmacrop select_join_fields(q, query, list, table_name) do + quote do + Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q -> + select_merge( + q, + ^%{ + shortname(unquote(query), metric_or_dimension) => + dynamic( + [e, s], + selected_as( + field(unquote(table_name), ^shortname(unquote(query), metric_or_dimension)), + ^shortname(unquote(query), metric_or_dimension) + ) + ) + } + ) + end) + end + end + + # Adds conversion_rate metric to query, calculated as + # X / Y where Y is the same breakdown value without goal or props + # filters. + def maybe_add_global_conversion_rate(q, site, query) do + if :conversion_rate in query.metrics do + total_query = + query + |> Query.remove_filters(["event:goal", "event:props"]) + |> Query.set_dimensions([]) + + q + |> select_merge( + ^%{ + total_visitors: Base.total_visitors_subquery(site, total_query, query.include_imported) + } + ) + |> select_merge([e], %{ + conversion_rate: + selected_as( + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + selected_as(:__total_visitors), + selected_as(:visitors), + selected_as(:__total_visitors) + ), + :conversion_rate + ) + }) + else + q + end + end + + # This function injects a group_conversion_rate metric into + # a dimensional query. It is calculated as X / Y, where: + # + # * X is the number of conversions for a set of dimensions + # result (conversion = number of visitors who + # completed the filtered goal with the filtered + # custom properties). + # + # * Y is the number of all visitors for this set of dimensions + # result without the `event:goal` and `event:props:*` + # filters. + def maybe_add_group_conversion_rate(q, site, query) do + if :group_conversion_rate in query.metrics do + group_totals_query = + query + |> Query.remove_filters(["event:goal", "event:props"]) + |> Query.set_metrics([:visitors]) + |> Query.set_order_by([]) + + from(e in subquery(q), + left_join: c in subquery(build(group_totals_query, site)), + on: ^build_group_by_join(query), + select_merge: %{ + total_visitors: c.visitors, + group_conversion_rate: + selected_as( + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + c.visitors, + e.visitors, + c.visitors + ), + :group_conversion_rate + ) + } + ) + |> select_join_fields(query, query.dimensions, e) + |> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e) + else + q + end + end + + defp join_query_results({nil, _}, {nil, _}), do: nil + + defp join_query_results({events_q, events_query}, {nil, _}), + do: events_q |> build_order_by(events_query) + + defp join_query_results({nil, events_query}, {sessions_q, _}), + do: sessions_q |> build_order_by(events_query) + + defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do + join(subquery(events_q), :left, [e], s in subquery(sessions_q), + on: ^build_group_by_join(events_query) + ) + |> select_join_fields(events_query, events_query.dimensions, e) + |> select_join_fields(events_query, events_query.metrics, e) + |> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s) + |> build_order_by(events_query) + end + + def build_group_by_join(%Query{dimensions: []}), do: true + + def build_group_by_join(query) do + query.dimensions + |> Enum.map(fn dim -> + dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim))) + end) + |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end) + end +end diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex index 8b77ca59288a..7f3efcf25e44 100644 --- a/lib/plausible/stats/table_decider.ex +++ b/lib/plausible/stats/table_decider.ex @@ -9,10 +9,12 @@ defmodule Plausible.Stats.TableDecider do alias Plausible.Stats.Query def events_join_sessions?(query) do - Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session)) + query + |> filter_keys() + |> Enum.any?(&(filters_partitioner(query, &1) == :session)) end - def partition_metrics(metrics, query, breakdown_property \\ nil) do + def partition_metrics(metrics, query) do %{ event: event_only_metrics, session: session_only_metrics, @@ -22,40 +24,52 @@ defmodule Plausible.Stats.TableDecider do } = partition(metrics, query, &metric_partitioner/2) - # Treat breakdown property as yet another filter - query = - if breakdown_property do - Query.put_filter(query, [:is, breakdown_property, []]) - else - query - end - %{event: event_only_filters, session: session_only_filters} = - partition(query.filters, query, &filters_partitioner/2) + query + |> filter_keys() + |> partition(query, &filters_partitioner/2) + + %{event: event_only_dimensions, session: session_only_dimensions} = + partition(query.dimensions, query, &filters_partitioner/2) cond do # Only one table needs to be queried - empty?(event_only_metrics) && empty?(event_only_filters) -> + empty?(event_only_metrics) && empty?(event_only_filters) && empty?(event_only_dimensions) -> {[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics} - empty?(session_only_metrics) && empty?(session_only_filters) -> + empty?(session_only_metrics) && empty?(session_only_filters) && + empty?(session_only_dimensions) -> {event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics} - # Filters on both events and sessions, but only one kind of metric - empty?(event_only_metrics) -> + # Filters and/or dimensions on both events and sessions, but only one kind of metric + empty?(event_only_metrics) && empty?(event_only_dimensions) -> {[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics} - empty?(session_only_metrics) -> + empty?(session_only_metrics) && empty?(session_only_dimensions) -> {event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics} - # Default: prefer sessions + # Default: prefer events true -> - {event_only_metrics ++ sample_percent, - session_only_metrics ++ either_metrics ++ sample_percent, other_metrics} + {event_only_metrics ++ either_metrics ++ sample_percent, + session_only_metrics ++ sample_percent, other_metrics} end end + defp filter_keys(query) do + query.filters + |> Enum.map(fn [_, filter_key | _rest] -> filter_key end) + end + + defp metric_partitioner(%Query{v2: true}, :conversion_rate), do: :either + defp metric_partitioner(%Query{v2: true}, :group_conversion_rate), do: :either + defp metric_partitioner(%Query{v2: true}, :visitors), do: :either + defp metric_partitioner(%Query{v2: true}, :visits), do: :either + # Note: This is inaccurate when filtering but required for old backwards compatibility + defp metric_partitioner(%Query{legacy_breakdown: true}, :pageviews), do: :either + defp metric_partitioner(%Query{legacy_breakdown: true}, :events), do: :either + defp metric_partitioner(_, :conversion_rate), do: :event + defp metric_partitioner(_, :group_conversion_rate), do: :event defp metric_partitioner(_, :average_revenue), do: :event defp metric_partitioner(_, :total_revenue), do: :event defp metric_partitioner(_, :pageviews), do: :event @@ -73,7 +87,7 @@ defmodule Plausible.Stats.TableDecider do # Calculated metrics - handled on callsite separately from other metrics. defp metric_partitioner(_, :time_on_page), do: :other defp metric_partitioner(_, :total_visitors), do: :other - defp metric_partitioner(_, :percentage), do: :other + defp metric_partitioner(_, :percentage), do: :either # Sample percentage is included in both tables if queried. defp metric_partitioner(_, :sample_percent), do: :sample_percent @@ -83,16 +97,16 @@ defmodule Plausible.Stats.TableDecider do defp metric_partitioner(_, _), do: :either - defp filters_partitioner(_, [_, "event:" <> _ | _rest]), do: :event - defp filters_partitioner(_, [_, "visit:entry_page" | _rest]), do: :session - defp filters_partitioner(_, [_, "visit:entry_page_hostname" | _rest]), do: :session - defp filters_partitioner(_, [_, "visit:exit_page" | _rest]), do: :session - defp filters_partitioner(_, [_, "visit:exit_page_hostname" | _rest]), do: :session + defp filters_partitioner(_, "event:" <> _), do: :event + defp filters_partitioner(_, "visit:entry_page"), do: :session + defp filters_partitioner(_, "visit:entry_page_hostname"), do: :session + defp filters_partitioner(_, "visit:exit_page"), do: :session + defp filters_partitioner(_, "visit:exit_page_hostname"), do: :session - defp filters_partitioner(%Query{experimental_reduced_joins?: true}, [_, "visit:" <> _ | _rest]), + defp filters_partitioner(%Query{experimental_reduced_joins?: true}, "visit:" <> _), do: :either - defp filters_partitioner(_, [_, "visit:" <> _ | _rest]), + defp filters_partitioner(_, "visit:" <> _), do: :session defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.ex index 85c36888f898..60bdd7a931ab 100644 --- a/lib/plausible/stats/util.ex +++ b/lib/plausible/stats/util.ex @@ -41,7 +41,8 @@ defmodule Plausible.Stats.Util do for any of the other metrics to be calculated. """ def maybe_add_visitors_metric(metrics) do - needed? = Enum.any?([:conversion_rate, :time_on_page], &(&1 in metrics)) + needed? = + Enum.any?([:conversion_rate, :group_conversion_rate, :time_on_page], &(&1 in metrics)) if needed? and :visitors not in metrics do metrics ++ [:visitors] @@ -49,4 +50,12 @@ defmodule Plausible.Stats.Util do metrics end end + + def shortname(_query, metric) when is_atom(metric), do: metric + def shortname(_query, "time:" <> _), do: :time + + def shortname(query, dimension) do + index = Enum.find_index(query.dimensions, &(&1 == dimension)) + :"dim#{index}" + end end diff --git a/lib/plausible/verification/checks/snippet.ex b/lib/plausible/verification/checks/snippet.ex index 9375a9112239..c1fbcfc2e043 100644 --- a/lib/plausible/verification/checks/snippet.ex +++ b/lib/plausible/verification/checks/snippet.ex @@ -11,8 +11,8 @@ defmodule Plausible.Verification.Checks.Snippet do @impl true def perform(%State{assigns: %{document: document}} = state) do - in_head = Floki.find(document, "head script[data-domain]") - in_body = Floki.find(document, "body script[data-domain]") + in_head = Floki.find(document, "head script[data-domain][src]") + in_body = Floki.find(document, "body script[data-domain][src]") all = in_head ++ in_body diff --git a/lib/plausible/verification/diagnostics.ex b/lib/plausible/verification/diagnostics.ex index 2465c25496ba..d89fc65d38ad 100644 --- a/lib/plausible/verification/diagnostics.ex +++ b/lib/plausible/verification/diagnostics.ex @@ -50,6 +50,13 @@ defmodule Plausible.Verification.Diagnostics do %Result{ok?: true} end + def interpret( + %__MODULE__{plausible_installed?: false, gtm_likely?: true, disallowed_via_csp?: true}, + _url + ) do + error(@errors.csp) + end + def interpret( %__MODULE__{plausible_installed?: false, gtm_likely?: true, cookie_banner_likely?: true}, _url @@ -87,6 +94,21 @@ defmodule Plausible.Verification.Diagnostics do error(@errors.no_snippet) end + def interpret( + %__MODULE__{ + plausible_installed?: true, + snippets_found_in_head: 0, + snippets_found_in_body: 0, + body_fetched?: true, + gtm_likely?: false, + callback_status: callback_status + }, + _url + ) + when is_integer(callback_status) and callback_status > 202 do + error(@errors.no_snippet) + end + def interpret( %__MODULE__{ plausible_installed?: false, @@ -111,6 +133,16 @@ defmodule Plausible.Verification.Diagnostics do error(@errors.unreachable) end + def interpret( + %__MODULE__{ + plausible_installed?: false, + service_error: :timeout + }, + _url + ) do + error(@errors.generic) + end + def interpret( %__MODULE__{ plausible_installed?: false, @@ -141,7 +173,7 @@ defmodule Plausible.Verification.Diagnostics do }, _url ) do - error(@errors.old_script) + error(@errors.generic) end def interpret( diff --git a/lib/plausible/verification/errors.ex b/lib/plausible/verification/errors.ex index 324a70a3e188..152a364fb733 100644 --- a/lib/plausible/verification/errors.ex +++ b/lib/plausible/verification/errors.ex @@ -46,12 +46,12 @@ defmodule Plausible.Verification.Errors do url: "https://plausible.io/docs/troubleshoot-integration#how-to-manually-check-your-integration" }, - old_script: %{ - message: "We couldn't verify your website", + generic: %{ + message: "We couldn't automatically verify your website", recommendation: - "You're running an older version of our script so we cannot verify it automatically. Please update to the latest script", + "Please manually check your integration by following the instructions provided", url: - "https://plausible.io/docs/troubleshoot-integration#are-you-using-an-older-version-of-our-script" + "https://plausible.io/docs/troubleshoot-integration#how-to-manually-check-your-integration" }, old_script_wp_no_plugin: %{ message: "We couldn't verify your website", diff --git a/lib/plausible_release.ex b/lib/plausible_release.ex index 9d23382e765e..e4c39caf47d0 100644 --- a/lib/plausible_release.ex +++ b/lib/plausible_release.ex @@ -85,7 +85,7 @@ defmodule Plausible.Release do plans = Plausible.Billing.Plans.all() - |> Plausible.Billing.Plans.with_prices() + |> Plausible.Billing.Plans.with_prices("127.0.0.1") |> Enum.map(fn plan -> plan = Map.from_struct(plan) diff --git a/lib/plausible_web/components/billing/billing.ex b/lib/plausible_web/components/billing/billing.ex index d3126bd2dd60..299064d2f214 100644 --- a/lib/plausible_web/components/billing/billing.ex +++ b/lib/plausible_web/components/billing/billing.ex @@ -16,12 +16,12 @@ defmodule PlausibleWeb.Components.Billing do def render_monthly_pageview_usage(assigns) do ~H""" -
+

Monthly pageviews usage

    <.billing_cycle_tab - name="Ongoing cycle" + name="Upcoming cycle" tab={:current_cycle} date_range={@usage.current_cycle.date_range} with_separator={true} @@ -30,7 +30,6 @@ defmodule PlausibleWeb.Components.Billing do name="Last cycle" tab={:last_cycle} date_range={@usage.last_cycle.date_range} - disabled={@usage.last_cycle.total == 0 && @usage.penultimate_cycle.total == 0} with_separator={true} /> <.billing_cycle_tab diff --git a/lib/plausible_web/components/billing/notice.ex b/lib/plausible_web/components/billing/notice.ex index 178d71aa9093..4856b87beafc 100644 --- a/lib/plausible_web/components/billing/notice.ex +++ b/lib/plausible_web/components/billing/notice.ex @@ -217,6 +217,31 @@ defmodule PlausibleWeb.Components.Billing.Notice do """ end + def pending_site_ownerships_notice(%{pending_ownership_count: count} = assigns) do + if count > 0 do + message = + "Your account has been invited to become the owner of " <> + if(count == 1, do: "a site, which is", else: "#{count} sites, which are") <> + " being counted towards the usage of your account." + + assigns = assign(assigns, message: message) + + ~H""" + + """ + else + ~H"" + end + end + def growth_grandfathered(assigns) do ~H"""
    diff --git a/lib/plausible_web/components/billing/plan_box.ex b/lib/plausible_web/components/billing/plan_box.ex index 574a0f046c0c..887b29ae35c3 100644 --- a/lib/plausible_web/components/billing/plan_box.ex +++ b/lib/plausible_web/components/billing/plan_box.ex @@ -168,7 +168,7 @@ defmodule PlausibleWeb.Components.Billing.PlanBox do {checkout_disabled, disabled_message} = cond do - not assigns.eligible_for_upgrade? -> + not Quota.eligible_for_upgrade?(assigns.usage) -> {true, nil} change_plan_link_text == "Currently on this plan" && not subscription_deleted -> @@ -330,7 +330,7 @@ defmodule PlausibleWeb.Components.Billing.PlanBox do defp losing_features_message(features_to_lose) do features_list_str = features_to_lose - |> Enum.map(& &1.display_name) + |> Enum.map(fn feature_mod -> feature_mod.display_name() end) |> PlausibleWeb.TextHelpers.pretty_join() "This plan does not support #{features_list_str}, which you are currently using. Please note that by subscribing to this plan you will lose access to #{if length(features_to_lose) == 1, do: "this feature", else: "these features"}." diff --git a/lib/plausible_web/controllers/admin_controller.ex b/lib/plausible_web/controllers/admin_controller.ex index d10bab1a5385..fd68c75bde87 100644 --- a/lib/plausible_web/controllers/admin_controller.ex +++ b/lib/plausible_web/controllers/admin_controller.ex @@ -9,47 +9,87 @@ defmodule PlausibleWeb.AdminController do |> String.to_integer() |> Plausible.Users.with_subscription() - usage = Quota.usage(user, with_features: true) + usage = Quota.Usage.usage(user, with_features: true) limits = %{ - monthly_pageviews: Quota.monthly_pageview_limit(user), - sites: Quota.site_limit(user), - team_members: Quota.team_member_limit(user) + monthly_pageviews: Quota.Limits.monthly_pageview_limit(user), + sites: Quota.Limits.site_limit(user), + team_members: Quota.Limits.team_member_limit(user) } - html_response = usage_and_limits_html(user, usage, limits) + html_response = usage_and_limits_html(user, usage, limits, params["embed"] == "true") conn |> put_resp_content_type("text/html") |> send_resp(200, html_response) end - defp usage_and_limits_html(user, usage, limits) do - """ - - - - - - - Usage - user:#{user.id} - - - - + def current_plan(conn, params) do + user = + params["user_id"] + |> String.to_integer() + |> Plausible.Users.with_subscription() + + plan = + case user && user.subscription && + Plausible.Billing.Plans.get_subscription_plan(user.subscription) do + %{} = plan -> + plan + |> Map.take([ + :billing_interval, + :monthly_pageview_limit, + :site_limit, + :team_member_limit, + :hourly_api_request_limit, + :features + ]) + |> Map.update(:features, [], fn features -> Enum.map(features, & &1.name()) end) + + _ -> + %{features: []} + end + + json_response = Jason.encode!(plan) + + conn + |> put_resp_content_type("application/json") + |> send_resp(200, json_response) + end + + defp usage_and_limits_html(user, usage, limits, embed?) do + content = """
    • Sites: #{usage.sites} / #{limits.sites}
    • Team members: #{usage.team_members} / #{limits.team_members}
    • Features: #{features_usage(usage.features)}
    • Monthly pageviews: #{monthly_pageviews_usage(usage.monthly_pageviews, limits.monthly_pageviews)}
    - - - """ + + if embed? do + content + else + """ + + + + + + + Usage - user:#{user.id} + + + + + #{content} + + + + """ + end end defp features_usage(features_module_list) do diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.ex new file mode 100644 index 000000000000..d2bf01401f14 --- /dev/null +++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex @@ -0,0 +1,23 @@ +defmodule PlausibleWeb.Api.ExternalQueryApiController do + @moduledoc false + + use PlausibleWeb, :controller + use Plausible.Repo + use PlausibleWeb.Plugs.ErrorHandler + alias Plausible.Stats.Query + + def query(conn, params) do + site = Repo.preload(conn.assigns.site, :owner) + + case Query.build(site, params) do + {:ok, query} -> + results = Plausible.Stats.query(site, query) + json(conn, results) + + {:error, message} -> + conn + |> put_status(400) + |> json(%{error: message}) + end + end +end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index f8145b8ccfa0..248f709baf13 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -124,14 +124,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController do prop_filter = Query.get_filter_by_prefix(query, "event:props:") query_allowed? = - case {prop_filter, query.property, allowed_props} do + case {prop_filter, query.dimensions, allowed_props} do {_, _, :all} -> true {[_, "event:props:" <> prop | _], _property, allowed_props} -> prop in allowed_props - {_filter, "event:props:" <> prop, allowed_props} -> + {_filter, ["event:props:" <> prop], allowed_props} -> prop in allowed_props _ -> @@ -171,10 +171,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do Query.get_filter(query, "event:name") -> {:error, "Metric `#{metric}` cannot be queried when filtering by `event:name`"} - query.property == "event:page" -> + query.dimensions == ["event:page"] -> {:ok, metric} - not is_nil(query.property) -> + not Enum.empty?(query.dimensions) -> {:error, "Metric `#{metric}` is not supported in breakdown queries (except `event:page` breakdown)"} @@ -189,7 +189,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do defp validate_metric("conversion_rate" = metric, query) do cond do - query.property == "event:goal" -> + query.dimensions == ["event:goal"] -> {:ok, metric} Query.get_filter(query, "event:goal") -> @@ -210,7 +210,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do Query.get_filter(query, "event:page") -> {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."} - query.property != nil -> + not Enum.empty?(query.dimensions) -> {:error, "Metric `#{metric}` is not supported in breakdown queries."} true -> @@ -230,9 +230,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController do defp validate_session_metric(metric, query) do cond do - event_only_property?(query.property) -> + length(query.dimensions) == 1 and event_only_property?(hd(query.dimensions)) -> {:error, - "Session metric `#{metric}` cannot be queried for breakdown by `#{query.property}`."} + "Session metric `#{metric}` cannot be queried for breakdown by `#{query.dimensions}`."} event_only_filter = find_event_only_filter(query) -> {:error, diff --git a/lib/plausible_web/controllers/api/paddle_controller.ex b/lib/plausible_web/controllers/api/paddle_controller.ex index 44be8ade60e5..bd3e8b0b40c8 100644 --- a/lib/plausible_web/controllers/api/paddle_controller.ex +++ b/lib/plausible_web/controllers/api/paddle_controller.ex @@ -3,7 +3,7 @@ defmodule PlausibleWeb.Api.PaddleController do use Plausible.Repo require Logger - plug :verify_signature + plug :verify_signature when action in [:webhook] def webhook(conn, %{"alert_name" => "subscription_created"} = params) do Plausible.Billing.subscription_created(params) @@ -29,6 +29,42 @@ defmodule PlausibleWeb.Api.PaddleController do send_resp(conn, 404, "") |> halt end + @default_currency_fallback :EUR + + def currency(conn, _params) do + plan_id = get_currency_reference_plan_id() + customer_ip = PlausibleWeb.RemoteIP.get(conn) + + result = + Plausible.Cache.Adapter.fetch(:customer_currency, {plan_id, customer_ip}, fn -> + case Plausible.Billing.PaddleApi.fetch_prices([plan_id], customer_ip) do + {:ok, %{^plan_id => money}} -> + {:ok, money.currency} + + error -> + Sentry.capture_message("Failed to fetch currency reference plan", + extra: %{error: inspect(error)} + ) + + {:error, :fetch_prices_failed} + end + end) + + case result do + {:ok, currency} -> + conn + |> put_status(200) + |> json(%{currency: Cldr.Currency.currency_for_code!(currency).narrow_symbol}) + + {:error, :fetch_prices_failed} -> + conn + |> put_status(200) + |> json(%{ + currency: Cldr.Currency.currency_for_code!(@default_currency_fallback).narrow_symbol + }) + end + end + def verify_signature(conn, _opts) do signature = Base.decode64!(conn.params["p_signature"]) @@ -49,18 +85,14 @@ defmodule PlausibleWeb.Api.PaddleController do end end - def verified_signature?(params) do - signature = Base.decode64!(params["p_signature"]) - - msg = - Map.delete(params, "p_signature") - |> Enum.map(fn {key, val} -> {key, "#{val}"} end) - |> List.keysort(0) - |> PhpSerializer.serialize() - - [key_entry] = :public_key.pem_decode(get_paddle_key()) - public_key = :public_key.pem_entry_decode(key_entry) - :public_key.verify(msg, :sha, signature, public_key) + @paddle_currency_reference_plan_id "857097" + @paddle_sandbox_currency_reference_plan_id "63842" + defp get_currency_reference_plan_id() do + if Application.get_env(:plausible, :environment) in ["dev", "staging"] do + @paddle_sandbox_currency_reference_plan_id + else + @paddle_currency_reference_plan_id + end end @paddle_prod_key File.read!("priv/paddle.pem") diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 6e0907147720..5dcf7800cf63 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -6,7 +6,7 @@ defmodule PlausibleWeb.Api.StatsController do alias Plausible.Stats alias Plausible.Stats.{Query, Comparisons} - alias Plausible.Stats.Filters.DashboardFilterParser + alias Plausible.Stats.Filters.LegacyDashboardFilterParser alias PlausibleWeb.Api.Helpers, as: H require Logger @@ -769,7 +769,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] params = Map.put(params, "property", "visit:referrer") - referrer_filter = DashboardFilterParser.filter_value("visit:source", referrer) + referrer_filter = LegacyDashboardFilterParser.filter_value("visit:source", referrer) query = Query.from(site, params) @@ -903,9 +903,9 @@ defmodule PlausibleWeb.Api.StatsController do total_pageviews_query = query |> Query.remove_filters(["visit:exit_page"]) - |> Query.put_filter([:member, "event:page", pages]) - |> Query.put_filter([:is, "event:name", "pageview"]) - |> Query.set_property("event:page") + |> Query.put_filter([:is, "event:page", pages]) + |> Query.put_filter([:is, "event:name", ["pageview"]]) + |> Query.set_dimensions(["event:page"]) total_pageviews = Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1}) diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index 1a526c09e1e8..c9eff5765c7b 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -633,12 +633,12 @@ defmodule PlausibleWeb.AuthController do subscription: user.subscription, invoices: Plausible.Billing.paddle_api().get_invoices(user.subscription), theme: user.theme || "system", - team_member_limit: Quota.team_member_limit(user), - team_member_usage: Quota.team_member_usage(user), - site_limit: Quota.site_limit(user), - site_usage: Quota.site_usage(user), - pageview_limit: Quota.monthly_pageview_limit(user), - pageview_usage: Quota.monthly_pageview_usage(user), + team_member_limit: Quota.Limits.team_member_limit(user), + team_member_usage: Quota.Usage.team_member_usage(user), + site_limit: Quota.Limits.site_limit(user), + site_usage: Quota.Usage.site_usage(user), + pageview_limit: Quota.Limits.monthly_pageview_limit(user), + pageview_usage: Quota.Usage.monthly_pageview_usage(user), totp_enabled?: Auth.TOTP.enabled?(user) ) end diff --git a/lib/plausible_web/controllers/billing_controller.ex b/lib/plausible_web/controllers/billing_controller.ex index 6069e2958b1d..c785c265fc59 100644 --- a/lib/plausible_web/controllers/billing_controller.ex +++ b/lib/plausible_web/controllers/billing_controller.ex @@ -31,7 +31,8 @@ defmodule PlausibleWeb.BillingController do def upgrade_to_enterprise_plan(conn, _params) do user = Plausible.Users.with_subscription(conn.assigns.current_user) - {latest_enterprise_plan, price} = Plans.latest_enterprise_plan_with_price(user) + {latest_enterprise_plan, price} = + Plans.latest_enterprise_plan_with_price(user, PlausibleWeb.RemoteIP.get(conn)) subscription_resumable? = Plausible.Billing.Subscriptions.resumable?(user.subscription) diff --git a/lib/plausible_web/controllers/site/membership_controller.ex b/lib/plausible_web/controllers/site/membership_controller.ex index 65361f3d8960..8cd5101d8c7b 100644 --- a/lib/plausible_web/controllers/site/membership_controller.ex +++ b/lib/plausible_web/controllers/site/membership_controller.ex @@ -29,8 +29,8 @@ defmodule PlausibleWeb.Site.MembershipController do |> Sites.get_for_user!(conn.assigns.site.domain) |> Plausible.Repo.preload(:owner) - limit = Plausible.Billing.Quota.team_member_limit(site.owner) - usage = Plausible.Billing.Quota.team_member_usage(site.owner) + limit = Plausible.Billing.Quota.Limits.team_member_limit(site.owner) + usage = Plausible.Billing.Quota.Usage.team_member_usage(site.owner) below_limit? = Plausible.Billing.Quota.below_limit?(usage, limit) render( diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 887c68d0ebf5..6082151a855b 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -18,8 +18,8 @@ defmodule PlausibleWeb.SiteController do render(conn, "new.html", changeset: Plausible.Site.changeset(%Plausible.Site{}), - first_site?: Quota.site_usage(current_user) == 0, - site_limit: Quota.site_limit(current_user), + first_site?: Quota.Usage.site_usage(current_user) == 0, + site_limit: Quota.Limits.site_limit(current_user), site_limit_exceeded?: Quota.ensure_can_add_new_site(current_user) != :ok, layout: {PlausibleWeb.LayoutView, "focus.html"} ) @@ -27,7 +27,7 @@ defmodule PlausibleWeb.SiteController do def create_site(conn, %{"site" => site_params}) do user = conn.assigns[:current_user] - first_site? = Quota.site_usage(user) == 0 + first_site? = Quota.Usage.site_usage(user) == 0 case Sites.create(user, site_params) do {:ok, %{site: site}} -> @@ -53,7 +53,7 @@ defmodule PlausibleWeb.SiteController do render(conn, "new.html", changeset: changeset, first_site?: first_site?, - site_limit: Quota.site_limit(user), + site_limit: Quota.Limits.site_limit(user), site_limit_exceeded?: false, layout: {PlausibleWeb.LayoutView, "focus.html"} ) diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index 1325fe642a93..4100cd7aecaa 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -55,7 +55,7 @@ defmodule PlausibleWeb.StatsController do stats_start_date = Plausible.Sites.stats_start_date(site) can_see_stats? = not Sites.locked?(site) or conn.assigns[:current_user_role] == :super_admin demo = site.domain == PlausibleWeb.Endpoint.host() - dogfood_page_path = if !demo, do: "/:dashboard" + dogfood_page_path = if demo, do: "/#{site.domain}", else: "/:dashboard" skip_to_dashboard? = conn.params["skip_to_dashboard"] == "true" cond do diff --git a/lib/plausible_web/live/choose_plan.ex b/lib/plausible_web/live/choose_plan.ex index 39d2866afead..1946b9c39c20 100644 --- a/lib/plausible_web/live/choose_plan.ex +++ b/lib/plausible_web/live/choose_plan.ex @@ -10,25 +10,30 @@ defmodule PlausibleWeb.Live.ChoosePlan do alias PlausibleWeb.Components.Billing.{PlanBox, PlanBenefits, Notice, PageviewSlider} alias Plausible.Site alias Plausible.Users - alias Plausible.Billing.{Plans, Plan, Quota} + alias Plausible.Billing.{Plans, Quota} @contact_link "https://plausible.io/contact" @billing_faq_link "https://plausible.io/docs/billing" - def mount(_params, %{"current_user_id" => user_id}, socket) do + def mount(_params, %{"current_user_id" => user_id, "remote_ip" => remote_ip}, socket) do socket = socket |> assign_new(:user, fn -> Users.with_subscription(user_id) end) - |> assign_new(:usage, fn %{user: user} -> - Quota.usage(user, with_features: true) + |> assign_new(:pending_ownership_site_ids, fn %{user: user} -> + user.email + |> Site.Memberships.all_pending_ownerships() + |> Enum.map(& &1.site_id) end) - |> assign_new(:last_30_days_usage, fn %{user: user, usage: usage} -> - case usage do - %{last_30_days: usage_cycle} -> usage_cycle.total - _ -> Quota.usage_cycle(user, :last_30_days).total - end + |> assign_new(:usage, fn %{ + user: user, + pending_ownership_site_ids: pending_ownership_site_ids + } -> + Quota.Usage.usage(user, + with_features: true, + pending_ownership_site_ids: pending_ownership_site_ids + ) end) |> assign_new(:owned_plan, fn %{user: %{subscription: subscription}} -> Plans.get_regular_plan(subscription, only_non_expired: true) @@ -36,18 +41,12 @@ defmodule PlausibleWeb.Live.ChoosePlan do |> assign_new(:owned_tier, fn %{owned_plan: owned_plan} -> if owned_plan, do: Map.get(owned_plan, :kind), else: nil end) - |> assign_new(:eligible_for_upgrade?, fn %{user: user, usage: usage} -> - has_sites? = usage.sites > 0 - has_pending_ownerships? = Site.Memberships.pending_ownerships?(user.email) - - has_sites? or has_pending_ownerships? - end) |> assign_new(:recommended_tier, fn %{ owned_plan: owned_plan, - eligible_for_upgrade?: eligible_for_upgrade?, + usage: usage, user: user } -> - if owned_plan != nil or not eligible_for_upgrade? do + if owned_plan != nil or not Quota.eligible_for_upgrade?(usage) do nil else Plans.suggest_tier(user) @@ -57,17 +56,16 @@ defmodule PlausibleWeb.Live.ChoosePlan do current_user_subscription_interval(user.subscription) end) |> assign_new(:available_plans, fn %{user: user} -> - Plans.available_plans_for(user, with_prices: true) + Plans.available_plans_for(user, with_prices: true, customer_ip: remote_ip) end) |> assign_new(:available_volumes, fn %{available_plans: available_plans} -> get_available_volumes(available_plans) end) |> assign_new(:selected_volume, fn %{ - owned_plan: owned_plan, - last_30_days_usage: last_30_days_usage, + usage: usage, available_volumes: available_volumes } -> - default_selected_volume(owned_plan, last_30_days_usage, available_volumes) + default_selected_volume(usage.monthly_pageviews, available_volumes) end) |> assign_new(:selected_interval, fn %{current_interval: current_interval} -> current_interval || :monthly @@ -110,9 +108,13 @@ defmodule PlausibleWeb.Live.ChoosePlan do ~H"""
    + - +

    <%= if @owned_plan, @@ -149,8 +151,7 @@ defmodule PlausibleWeb.Live.ChoosePlan do

    - You have used <%= PlausibleWeb.AuthView.delimit_integer(@last_30_days_usage) %> - billable pageviews in the last 30 days + <.render_usage pageview_usage={@usage.monthly_pageviews} />

    <.pageview_limit_notice :if={!@owned_plan} /> <.help_links /> @@ -160,6 +161,22 @@ defmodule PlausibleWeb.Live.ChoosePlan do """ end + defp render_usage(assigns) do + case assigns.pageview_usage do + %{last_30_days: _} -> + ~H""" + You have used + <%= PlausibleWeb.AuthView.delimit_integer(@pageview_usage.last_30_days.total) %> billable pageviews in the last 30 days + """ + + %{last_cycle: _} -> + ~H""" + You have used + <%= PlausibleWeb.AuthView.delimit_integer(@pageview_usage.last_cycle.total) %> billable pageviews in the last billing cycle + """ + end + end + def handle_event("set_interval", %{"interval" => interval}, socket) do new_interval = case interval do @@ -189,10 +206,14 @@ defmodule PlausibleWeb.Live.ChoosePlan do )} end - defp default_selected_volume(%Plan{monthly_pageview_limit: limit}, _, _), do: limit + defp default_selected_volume(pageview_usage, available_volumes) do + total = + case pageview_usage do + %{last_30_days: usage} -> usage.total + %{last_cycle: usage} -> usage.total + end - defp default_selected_volume(_, last_30_days_usage, available_volumes) do - Enum.find(available_volumes, &(last_30_days_usage < &1)) || :enterprise + Enum.find(available_volumes, &(total < &1)) || :enterprise end defp current_user_subscription_interval(subscription) do diff --git a/lib/plausible_web/live/components/combo_box.ex b/lib/plausible_web/live/components/combo_box.ex index 7da6ae301c8a..01d94f6a8b02 100644 --- a/lib/plausible_web/live/components/combo_box.ex +++ b/lib/plausible_web/live/components/combo_box.ex @@ -120,7 +120,7 @@ defmodule PlausibleWeb.Live.Components.ComboBox do + <.live_component module={Modal} id="some-form-modal" :let={modal_unique_id}> <.live_component module={SomeForm} - id="some-form" + id={"some-form-#{modal_unique_id}"} on_save_form={ fn entry, socket -> send(self(), {:entry_added, entry}) @@ -42,7 +42,15 @@ defmodule PlausibleWeb.Live.Components.Modal do is called on it. On subsequent openings within the same session the contents of the modal are completely remounted. This assures that any stateful components inside the modal are reset to their - initial state. + initial state. The modal component provides `modal_unique_id` + as an argument to its inner block. Appending this ID to every + live components' ID nested inside the modal is important for + consistent state reset on every reopening. This also applies + to live components nested inside live components embedded directly + in the modal's inner block - then the unique ID should be also + passed down as an attribute and appended accordingly. Appending can + be skipped if embedded component handles state reset explicitly + (via, for instance, `phx-click-away` callback). `Modal` exposes two functions for managing window state: @@ -109,7 +117,13 @@ defmodule PlausibleWeb.Live.Components.Modal do assign(socket, id: assigns.id, inner_block: assigns.inner_block, - load_content?: true + # Initial value is constant, as dead view ID + # must match the ID after the connection is + # established. Otherwise, there will be problems + # with live components relying on ID for setup + # on mount (using AlpineJS, for instance). + load_content?: true, + modal_sequence_id: 0 ) {:ok, socket} @@ -144,7 +158,6 @@ defmodule PlausibleWeb.Live.Components.Modal do document.body.style['overflow-y'] = 'hidden'; if (this.firstLoadDone) { - liveSocket.execJS($el, $el.dataset.onclose); liveSocket.execJS($el, $el.dataset.onopen); } else { this.firstLoadDone = true; @@ -154,6 +167,7 @@ defmodule PlausibleWeb.Live.Components.Modal do }, closeModal() { this.modalOpen = false; + liveSocket.execJS($el, $el.dataset.onclose); setTimeout(function() { document.body.style['overflow-y'] = 'auto'; @@ -204,9 +218,9 @@ defmodule PlausibleWeb.Live.Components.Modal do x-transition:leave-end="opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95" x-on:click.outside="closeModal()" > - <%= render_slot(@inner_block) %> + <%= render_slot(@inner_block, modal_unique_id(@modal_sequence_id)) %> - """ @@ -296,7 +322,12 @@ defmodule PlausibleWeb.Live.GoalSettings.Form do end def handle_event("switch-tab", %{"tab" => tab}, socket) do - {:noreply, assign(socket, selected_tab: tab)} + socket = + socket + |> assign(:selected_tab, tab) + |> update(:tab_sequence_id, &(&1 + 1)) + + {:noreply, socket} end def handle_event("save-goal", %{"goal" => goal}, socket) do @@ -318,11 +349,26 @@ defmodule PlausibleWeb.Live.GoalSettings.Form do {:noreply, socket.assigns.on_autoconfigure.(socket)} end - def suggest_page_paths(input, _options, site) do + def suggest_page_paths(input, site) do query = Plausible.Stats.Query.from(site, %{"with_imported" => "true", "period" => "all"}) site |> Plausible.Stats.filter_suggestions(query, "page", input) |> Enum.map(fn %{label: label, value: value} -> {label, value} end) end + + def suggest_event_names(input, site, existing_goals) do + existing_names = + existing_goals + |> Enum.reject(&is_nil(&1.event_name)) + |> Enum.map(& &1.event_name) + + site + |> Plausible.Stats.GoalSuggestions.suggest_event_names(input, exclude: existing_names) + |> Enum.map(fn name -> {name, name} end) + end + + defp suffix(context_unique_id, tab_sequence_id) do + "#{context_unique_id}-tabseq#{tab_sequence_id}" + end end diff --git a/lib/plausible_web/live/register_form.ex b/lib/plausible_web/live/register_form.ex index 6e5462b6cc3f..111ae687b818 100644 --- a/lib/plausible_web/live/register_form.ex +++ b/lib/plausible_web/live/register_form.ex @@ -144,7 +144,9 @@ defmodule PlausibleWeb.Live.RegisterForm do >
    <%= if @captcha_error do %> -
    <%= @captcha_error %>
    +
    + <%= @captcha_error %> +
    <% end %> + + + Hello + + + + """ + test "no src attr doesn't count as snippet" do + stub_fetch_body(200, @no_src_scripts) + stub_installation(200, plausible_installed(false)) + + run_checks() + |> Checks.interpret_diagnostics() + |> assert_error(@errors.no_snippet) + end + @many_snippets_ok """ @@ -524,6 +544,21 @@ defmodule Plausible.Verification.ChecksTest do """ + test "disallowd via content-security-policy and GTM should make CSP a priority" do + stub_fetch_body(fn conn -> + conn + |> put_resp_header("content-security-policy", "default-src 'self' foo.local") + |> put_resp_content_type("text/html") + |> send_resp(200, @gtm_body) + end) + + stub_installation(200, plausible_installed(false)) + + run_checks() + |> Checks.interpret_diagnostics() + |> assert_error(@errors.csp) + end + test "detecting gtm" do stub_fetch_body(200, @gtm_body) stub_installation(200, plausible_installed(false)) @@ -689,7 +724,7 @@ defmodule Plausible.Verification.ChecksTest do run_checks() |> Checks.interpret_diagnostics() - |> assert_error(@errors.old_script) + |> assert_error(@errors.generic) end test "callback handling not found for wordpress site" do @@ -773,6 +808,52 @@ defmodule Plausible.Verification.ChecksTest do |> interpret_sentry_case() |> assert_error(@errors.old_script_wp_no_plugin) end + + test "service timeout" do + %Plausible.Verification.Diagnostics{ + plausible_installed?: false, + snippets_found_in_head: 1, + snippets_found_in_body: 0, + snippet_found_after_busting_cache?: false, + snippet_unknown_attributes?: false, + disallowed_via_csp?: false, + service_error: :timeout, + body_fetched?: true, + wordpress_likely?: true, + cookie_banner_likely?: false, + gtm_likely?: false, + callback_status: 0, + proxy_likely?: true, + manual_script_extension?: false, + data_domain_mismatch?: false, + wordpress_plugin?: false + } + |> interpret_sentry_case() + |> assert_error(@errors.generic) + end + + test "malformed snippet code, that headless somewhat accepts" do + %Plausible.Verification.Diagnostics{ + plausible_installed?: true, + snippets_found_in_head: 0, + snippets_found_in_body: 0, + snippet_found_after_busting_cache?: false, + snippet_unknown_attributes?: false, + disallowed_via_csp?: false, + service_error: nil, + body_fetched?: true, + wordpress_likely?: false, + cookie_banner_likely?: false, + gtm_likely?: false, + callback_status: 405, + proxy_likely?: false, + manual_script_extension?: false, + data_domain_mismatch?: false, + wordpress_plugin?: false + } + |> interpret_sentry_case() + |> assert_error(@errors.no_snippet) + end end defp interpret_sentry_case(diagnostics) do diff --git a/test/plausible/stats/dashboard_filter_parser_test.exs b/test/plausible/stats/dashboard_filter_parser_test.exs deleted file mode 100644 index 1602380f3780..000000000000 --- a/test/plausible/stats/dashboard_filter_parser_test.exs +++ /dev/null @@ -1,207 +0,0 @@ -defmodule Plausible.Stats.DashboardFilterParserTest do - use ExUnit.Case, async: true - alias Plausible.Stats.Filters.DashboardFilterParser - - def assert_parsed(filters, expected_output) do - assert DashboardFilterParser.parse_and_prefix(filters) == expected_output - end - - describe "adding prefix" do - test "adds appropriate prefix to filter" do - %{"page" => "/"} - |> assert_parsed([[:is, "event:page", "/"]]) - - %{"goal" => "Signup"} - |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]]) - - %{"goal" => "Visit /blog"} - |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]]) - - %{"source" => "Google"} - |> assert_parsed([[:is, "visit:source", "Google"]]) - - %{"referrer" => "cnn.com"} - |> assert_parsed([[:is, "visit:referrer", "cnn.com"]]) - - %{"utm_medium" => "search"} - |> assert_parsed([[:is, "visit:utm_medium", "search"]]) - - %{"utm_source" => "bing"} - |> assert_parsed([[:is, "visit:utm_source", "bing"]]) - - %{"utm_content" => "content"} - |> assert_parsed([[:is, "visit:utm_content", "content"]]) - - %{"utm_term" => "term"} - |> assert_parsed([[:is, "visit:utm_term", "term"]]) - - %{"screen" => "Desktop"} - |> assert_parsed([[:is, "visit:screen", "Desktop"]]) - - %{"browser" => "Opera"} - |> assert_parsed([[:is, "visit:browser", "Opera"]]) - - %{"browser_version" => "10.1"} - |> assert_parsed([[:is, "visit:browser_version", "10.1"]]) - - %{"os" => "Linux"} - |> assert_parsed([[:is, "visit:os", "Linux"]]) - - %{"os_version" => "13.0"} - |> assert_parsed([[:is, "visit:os_version", "13.0"]]) - - %{"country" => "EE"} - |> assert_parsed([[:is, "visit:country", "EE"]]) - - %{"region" => "EE-12"} - |> assert_parsed([[:is, "visit:region", "EE-12"]]) - - %{"city" => "123"} - |> assert_parsed([[:is, "visit:city", "123"]]) - - %{"entry_page" => "/blog"} - |> assert_parsed([[:is, "visit:entry_page", "/blog"]]) - - %{"exit_page" => "/blog"} - |> assert_parsed([[:is, "visit:exit_page", "/blog"]]) - - %{"props" => %{"cta" => "Top"}} - |> assert_parsed([[:is, "event:props:cta", "Top"]]) - - %{"hostname" => "dummy.site"} - |> assert_parsed([[:is, "event:hostname", "dummy.site"]]) - end - end - - describe "escaping pipe character" do - test "in simple is filter" do - %{"goal" => ~S(Foo \| Bar)} - |> assert_parsed([[:is, "event:goal", {:event, "Foo | Bar"}]]) - end - - test "in member filter" do - %{"page" => ~S(/|\|)} - |> assert_parsed([[:member, "event:page", ["/", "|"]]]) - end - end - - describe "is not filter type" do - test "simple is not filter" do - %{"page" => "!/"} - |> assert_parsed([[:is_not, "event:page", "/"]]) - - %{"props" => %{"cta" => "!Top"}} - |> assert_parsed([[:is_not, "event:props:cta", "Top"]]) - end - end - - describe "member filter type" do - test "simple member filter" do - %{"page" => "/|/blog"} - |> assert_parsed([[:member, "event:page", ["/", "/blog"]]]) - end - - test "escaping pipe character" do - %{"page" => "/|\\|"} - |> assert_parsed([[:member, "event:page", ["/", "|"]]]) - end - - test "mixed goals" do - %{"goal" => "Signup|Visit /thank-you"} - |> assert_parsed([[:member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]]) - - %{"goal" => "Visit /thank-you|Signup"} - |> assert_parsed([[:member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]]]) - end - end - - describe "matches_member filter type" do - test "parses matches_member filter type" do - %{"page" => "/|/blog**"} - |> assert_parsed([[:matches_member, "event:page", ["/", "/blog**"]]]) - end - - test "parses not_matches_member filter type" do - %{"page" => "!/|/blog**"} - |> assert_parsed([[:not_matches_member, "event:page", ["/", "/blog**"]]]) - end - end - - describe "contains filter type" do - test "single contains" do - %{"page" => "~blog"} - |> assert_parsed([[:matches, "event:page", "**blog**"]]) - end - - test "negated contains" do - %{"page" => "!~articles"} - |> assert_parsed([[:does_not_match, "event:page", "**articles**"]]) - end - - test "contains member" do - %{"page" => "~articles|blog"} - |> assert_parsed([[:matches_member, "event:page", ["**articles**", "**blog**"]]]) - end - - test "not contains member" do - %{"page" => "!~articles|blog"} - |> assert_parsed([[:not_matches_member, "event:page", ["**articles**", "**blog**"]]]) - end - end - - describe "not_member filter type" do - test "simple not_member filter" do - %{"page" => "!/|/blog"} - |> assert_parsed([[:not_member, "event:page", ["/", "/blog"]]]) - end - - test "mixed goals" do - %{"goal" => "!Signup|Visit /thank-you"} - |> assert_parsed([ - [:not_member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]] - ]) - - %{"goal" => "!Visit /thank-you|Signup"} - |> assert_parsed([ - [:not_member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]] - ]) - end - end - - describe "matches filter type" do - test "can be used with `goal` or `page` filters" do - %{"page" => "/blog/post-*"} - |> assert_parsed([[:matches, "event:page", "/blog/post-*"]]) - - %{"goal" => "Visit /blog/post-*"} - |> assert_parsed([[:matches, "event:goal", {:page, "/blog/post-*"}]]) - end - - test "other filters default to `is` even when wildcard is present" do - %{"country" => "Germa**"} - |> assert_parsed([[:is, "visit:country", "Germa**"]]) - end - end - - describe "does_not_match filter type" do - test "can be used with `page` filter" do - %{"page" => "!/blog/post-*"} - |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]]) - end - - test "other filters default to is_not even when wildcard is present" do - %{"country" => "!Germa**"} - |> assert_parsed([[:is_not, "visit:country", "Germa**"]]) - end - end - - describe "contains prefix filter type" do - test "can be used with any filter" do - %{"page" => "~/blog/post"} - |> assert_parsed([[:matches, "event:page", "**/blog/post**"]]) - - %{"source" => "~facebook"} - |> assert_parsed([[:matches, "visit:source", "**facebook**"]]) - end - end -end diff --git a/test/plausible/stats/filters_test.exs b/test/plausible/stats/filters_test.exs index aa5b77412f62..a99423cd54ec 100644 --- a/test/plausible/stats/filters_test.exs +++ b/test/plausible/stats/filters_test.exs @@ -12,70 +12,70 @@ defmodule Plausible.Stats.FiltersTest do describe "parses filter expression" do test "simple positive" do "event:name==pageview" - |> assert_parsed([[:is, "event:name", "pageview"]]) + |> assert_parsed([[:is, "event:name", ["pageview"]]]) end test "simple negative" do "event:name!=pageview" - |> assert_parsed([[:is_not, "event:name", "pageview"]]) + |> assert_parsed([[:is_not, "event:name", ["pageview"]]]) end test "whitespace is trimmed" do " event:name == pageview " - |> assert_parsed([[:is, "event:name", "pageview"]]) + |> assert_parsed([[:is, "event:name", ["pageview"]]]) end test "wildcard" do "event:page==/blog/post-*" - |> assert_parsed([[:matches, "event:page", "/blog/post-*"]]) + |> assert_parsed([[:matches, "event:page", ["/blog/post-*"]]]) end test "negative wildcard" do "event:page!=/blog/post-*" - |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]]) + |> assert_parsed([[:does_not_match, "event:page", ["/blog/post-*"]]]) end test "custom event goal" do "event:goal==Signup" - |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]]) + |> assert_parsed([[:is, "event:goal", [{:event, "Signup"}]]]) end test "pageview goal" do "event:goal==Visit /blog" - |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]]) + |> assert_parsed([[:is, "event:goal", [{:page, "/blog"}]]]) end - test "member" do + test "is" do "visit:country==FR|GB|DE" - |> assert_parsed([[:member, "visit:country", ["FR", "GB", "DE"]]]) + |> assert_parsed([[:is, "visit:country", ["FR", "GB", "DE"]]]) end test "member + wildcard" do "event:page==/blog**|/newsletter|/*/" - |> assert_parsed([[:matches, "event:page", "/blog**|/newsletter|/*/"]]) + |> assert_parsed([[:matches, "event:page", ["/blog**|/newsletter|/*/"]]]) end test "combined with \";\"" do "event:page==/blog**|/newsletter|/*/ ; visit:country==FR|GB|DE" |> assert_parsed([ - [:matches, "event:page", "/blog**|/newsletter|/*/"], - [:member, "visit:country", ["FR", "GB", "DE"]] + [:matches, "event:page", ["/blog**|/newsletter|/*/"]], + [:is, "visit:country", ["FR", "GB", "DE"]] ]) end test "escaping pipe character" do "utm_campaign==campaign \\| 1" - |> assert_parsed([[:is, "utm_campaign", "campaign | 1"]]) + |> assert_parsed([[:is, "utm_campaign", ["campaign | 1"]]]) end - test "escaping pipe character in member filter" do + test "escaping pipe character in is filter" do "utm_campaign==campaign \\| 1|campaign \\| 2" - |> assert_parsed([[:member, "utm_campaign", ["campaign | 1", "campaign | 2"]]]) + |> assert_parsed([[:is, "utm_campaign", ["campaign | 1", "campaign | 2"]]]) end - test "keeps escape characters in member + wildcard filter" do + test "keeps escape characters in is + wildcard filter" do "event:page==/**\\|page|/other/page" - |> assert_parsed([[:matches, "event:page", "/**\\|page|/other/page"]]) + |> assert_parsed([[:matches, "event:page", ["/**\\|page|/other/page"]]]) end test "gracefully fails to parse garbage" do @@ -98,4 +98,12 @@ defmodule Plausible.Stats.FiltersTest do |> assert_parsed([]) end end + + describe "parses filters list" do + test "simple" do + [["is", "event:name", ["pageview"]]] + |> Jason.encode!() + |> assert_parsed([[:is, "event:name", ["pageview"]]]) + end + end end diff --git a/test/plausible/stats/goal_suggestions_test.exs b/test/plausible/stats/goal_suggestions_test.exs index 16d3109cdc15..42a087b88b8a 100644 --- a/test/plausible/stats/goal_suggestions_test.exs +++ b/test/plausible/stats/goal_suggestions_test.exs @@ -65,6 +65,27 @@ defmodule Plausible.Stats.GoalSuggestionsTest do assert GoalSuggestions.suggest_event_names(site, "") == ["Signup"] end + test "ignores event names with either white space on either end or consisting only of whitespace", + %{site: site} do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:event, name: "Signup"), + build(:event, name: " Signup2"), + build(:event, name: " Signup2 "), + build(:event, name: "Signup2 "), + build(:event, name: " "), + build(:imported_custom_events, name: "Auth", visitors: 3), + build(:imported_custom_events, name: " Auth2", visitors: 3), + build(:imported_custom_events, name: " Auth2 ", visitors: 3), + build(:imported_custom_events, name: "Auth2 ", visitors: 3), + build(:imported_custom_events, name: " ", visitors: 3), + build(:pageview) + ]) + + assert GoalSuggestions.suggest_event_names(site, "") == ["Auth", "Signup"] + end + test "can exclude goals from being suggested", %{site: site} do populate_stats(site, [build(:event, name: "Signup")]) diff --git a/test/plausible/stats/legacy_dashboard_filter_parser_test.exs b/test/plausible/stats/legacy_dashboard_filter_parser_test.exs new file mode 100644 index 000000000000..8a3e94053a32 --- /dev/null +++ b/test/plausible/stats/legacy_dashboard_filter_parser_test.exs @@ -0,0 +1,201 @@ +defmodule Plausible.Stats.LegacyDashboardFilterParserTest do + use ExUnit.Case, async: true + alias Plausible.Stats.Filters.LegacyDashboardFilterParser + + def assert_parsed(filters, expected_output) do + assert LegacyDashboardFilterParser.parse_and_prefix(filters) == expected_output + end + + describe "adding prefix" do + test "adds appropriate prefix to filter" do + %{"page" => "/"} + |> assert_parsed([[:is, "event:page", ["/"]]]) + + %{"goal" => "Signup"} + |> assert_parsed([[:is, "event:goal", [{:event, "Signup"}]]]) + + %{"goal" => "Visit /blog"} + |> assert_parsed([[:is, "event:goal", [{:page, "/blog"}]]]) + + %{"source" => "Google"} + |> assert_parsed([[:is, "visit:source", ["Google"]]]) + + %{"referrer" => "cnn.com"} + |> assert_parsed([[:is, "visit:referrer", ["cnn.com"]]]) + + %{"utm_medium" => "search"} + |> assert_parsed([[:is, "visit:utm_medium", ["search"]]]) + + %{"utm_source" => "bing"} + |> assert_parsed([[:is, "visit:utm_source", ["bing"]]]) + + %{"utm_content" => "content"} + |> assert_parsed([[:is, "visit:utm_content", ["content"]]]) + + %{"utm_term" => "term"} + |> assert_parsed([[:is, "visit:utm_term", ["term"]]]) + + %{"screen" => "Desktop"} + |> assert_parsed([[:is, "visit:screen", ["Desktop"]]]) + + %{"browser" => "Opera"} + |> assert_parsed([[:is, "visit:browser", ["Opera"]]]) + + %{"browser_version" => "10.1"} + |> assert_parsed([[:is, "visit:browser_version", ["10.1"]]]) + + %{"os" => "Linux"} + |> assert_parsed([[:is, "visit:os", ["Linux"]]]) + + %{"os_version" => "13.0"} + |> assert_parsed([[:is, "visit:os_version", ["13.0"]]]) + + %{"country" => "EE"} + |> assert_parsed([[:is, "visit:country", ["EE"]]]) + + %{"region" => "EE-12"} + |> assert_parsed([[:is, "visit:region", ["EE-12"]]]) + + %{"city" => "123"} + |> assert_parsed([[:is, "visit:city", ["123"]]]) + + %{"entry_page" => "/blog"} + |> assert_parsed([[:is, "visit:entry_page", ["/blog"]]]) + + %{"exit_page" => "/blog"} + |> assert_parsed([[:is, "visit:exit_page", ["/blog"]]]) + + %{"props" => %{"cta" => "Top"}} + |> assert_parsed([[:is, "event:props:cta", ["Top"]]]) + + %{"hostname" => "dummy.site"} + |> assert_parsed([[:is, "event:hostname", ["dummy.site"]]]) + end + end + + describe "escaping pipe character" do + test "in simple is filter" do + %{"goal" => ~S(Foo \| Bar)} + |> assert_parsed([[:is, "event:goal", [{:event, "Foo | Bar"}]]]) + end + + test "in member filter" do + %{"page" => ~S(/|\|)} + |> assert_parsed([[:is, "event:page", ["/", "|"]]]) + end + end + + describe "is not filter type" do + test "simple is not filter" do + %{"page" => "!/"} + |> assert_parsed([[:is_not, "event:page", ["/"]]]) + + %{"props" => %{"cta" => "!Top"}} + |> assert_parsed([[:is_not, "event:props:cta", ["Top"]]]) + end + end + + describe "is filter type" do + test "simple is filter" do + %{"page" => "/|/blog"} + |> assert_parsed([[:is, "event:page", ["/", "/blog"]]]) + end + + test "escaping pipe character" do + %{"page" => "/|\\|"} + |> assert_parsed([[:is, "event:page", ["/", "|"]]]) + end + + test "mixed goals" do + %{"goal" => "Signup|Visit /thank-you"} + |> assert_parsed([[:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]]) + + %{"goal" => "Visit /thank-you|Signup"} + |> assert_parsed([[:is, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]]]) + end + end + + describe "matches filter type" do + test "parses matches filter type" do + %{"page" => "/|/blog**"} + |> assert_parsed([[:matches, "event:page", ["/", "/blog**"]]]) + end + + test "parses not_matches filter type" do + %{"page" => "!/|/blog**"} + |> assert_parsed([[:does_not_match, "event:page", ["/", "/blog**"]]]) + end + + test "single matches" do + %{"page" => "~blog"} + |> assert_parsed([[:matches, "event:page", ["**blog**"]]]) + end + + test "negated matches" do + %{"page" => "!~articles"} + |> assert_parsed([[:does_not_match, "event:page", ["**articles**"]]]) + end + + test "matches member" do + %{"page" => "~articles|blog"} + |> assert_parsed([[:matches, "event:page", ["**articles**", "**blog**"]]]) + end + + test "not matches member" do + %{"page" => "!~articles|blog"} + |> assert_parsed([[:does_not_match, "event:page", ["**articles**", "**blog**"]]]) + end + + test "can be used with `goal` or `page` filters" do + %{"page" => "/blog/post-*"} + |> assert_parsed([[:matches, "event:page", ["/blog/post-*"]]]) + + %{"goal" => "Visit /blog/post-*"} + |> assert_parsed([[:matches, "event:goal", [{:page, "/blog/post-*"}]]]) + end + + test "other filters default to `is` even when wildcard is present" do + %{"country" => "Germa**"} + |> assert_parsed([[:is, "visit:country", ["Germa**"]]]) + end + + test "can be used with `page` filter" do + %{"page" => "!/blog/post-*"} + |> assert_parsed([[:does_not_match, "event:page", ["/blog/post-*"]]]) + end + + test "other filters default to is_not even when wildcard is present" do + %{"country" => "!Germa**"} + |> assert_parsed([[:is_not, "visit:country", ["Germa**"]]]) + end + end + + describe "is_not filter type" do + test "simple is_not filter" do + %{"page" => "!/|/blog"} + |> assert_parsed([[:is_not, "event:page", ["/", "/blog"]]]) + end + + test "mixed goals" do + %{"goal" => "!Signup|Visit /thank-you"} + |> assert_parsed([ + [:is_not, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]] + ]) + + %{"goal" => "!Visit /thank-you|Signup"} + |> assert_parsed([ + [:is_not, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]] + ]) + end + end + + describe "contains prefix filter type" do + test "can be used with any filter" do + %{"page" => "~/blog/post"} + |> assert_parsed([[:matches, "event:page", ["**/blog/post**"]]]) + + %{"source" => "~facebook"} + |> assert_parsed([[:matches, "visit:source", ["**facebook**"]]]) + end + end +end diff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exs new file mode 100644 index 000000000000..aee73b88c0e9 --- /dev/null +++ b/test/plausible/stats/query_optimizer_test.exs @@ -0,0 +1,147 @@ +defmodule Plausible.Stats.QueryOptimizerTest do + use Plausible.DataCase, async: true + + alias Plausible.Stats.{Query, QueryOptimizer} + + @default_params %{metrics: [:visitors]} + + def perform(params) do + params = Map.merge(@default_params, params) |> Map.to_list() + struct!(Query, params) |> QueryOptimizer.optimize() + end + + describe "add_missing_order_by" do + test "does nothing if order_by passed" do + assert perform(%{order_by: [visitors: :desc]}).order_by == [{:visitors, :desc}] + end + + test "adds first metric to order_by if order_by not specified" do + assert perform(%{metrics: [:pageviews, :visitors]}).order_by == [{:pageviews, :desc}] + + assert perform(%{metrics: [:pageviews, :visitors], dimensions: ["event:page"]}).order_by == + [{:pageviews, :desc}] + end + + test "adds time and first metric to order_by if order_by not specified" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-02-01 00:00:00]), + metrics: [:pageviews, :visitors], + dimensions: ["time", "event:page"] + }).order_by == + [{"time:day", :asc}, {:pageviews, :desc}] + end + end + + describe "update_group_by_time" do + test "does nothing if `time` dimension not passed" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-05 00:00:00]), + dimensions: ["time:month"] + }).dimensions == ["time:month"] + end + + test "updating time dimension" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + dimensions: ["time"] + }).dimensions == ["time:hour"] + + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 00:00:00]), + dimensions: ["time"] + }).dimensions == ["time:hour"] + + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 16:00:00]), + dimensions: ["time"] + }).dimensions == ["time:hour"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-01-04]), + dimensions: ["time"] + }).dimensions == ["time:day"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-01-10]), + dimensions: ["time"] + }).dimensions == ["time:day"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-01-16]), + dimensions: ["time"] + }).dimensions == ["time:day"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-02-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2023-11-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2024-01-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2026-01-01]), + dimensions: ["time"] + }).dimensions == ["time:month"] + end + end + + describe "update_time_in_order_by" do + test "updates explicit time dimension in order_by" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + dimensions: ["time:hour"], + order_by: [{"time", :asc}] + }).order_by == [{"time:hour", :asc}] + end + end + + describe "extend_hostname_filters_to_visit" do + test "updates filters it filtering by event:hostname and visit:referrer and visit:exit_page dimensions" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + filters: [ + [:is, "event:hostname", ["example.com"]], + [:matches, "event:hostname", ["*.com"]] + ], + dimensions: ["visit:referrer", "visit:exit_page"] + }).filters == [ + [:is, "event:hostname", ["example.com"]], + [:matches, "event:hostname", ["*.com"]], + [:is, "visit:entry_page_hostname", ["example.com"]], + [:matches, "visit:entry_page_hostname", ["*.com"]], + [:is, "visit:exit_page_hostname", ["example.com"]], + [:matches, "visit:exit_page_hostname", ["*.com"]] + ] + end + + test "does not update filters if not needed" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + filters: [ + [:is, "event:hostname", ["example.com"]] + ], + dimensions: ["time", "event:hostname"] + }).filters == [ + [:is, "event:hostname", ["example.com"]] + ] + end + end +end diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs new file mode 100644 index 000000000000..bf3f3bedb492 --- /dev/null +++ b/test/plausible/stats/query_parser_test.exs @@ -0,0 +1,731 @@ +defmodule Plausible.Stats.Filters.QueryParserTest do + use Plausible.DataCase + + alias Plausible.Stats.Filters + import Plausible.Stats.Filters.QueryParser + + setup [:create_user, :create_new_site] + + @today ~D[2021-05-05] + @date_range Date.range(@today, @today) + + def check_success(params, site, expected_result) do + assert parse(site, params, @today) == {:ok, expected_result} + end + + def check_error(params, site, expected_error_message) do + {:error, message} = parse(site, params, @today) + assert message =~ expected_error_message + end + + def check_date_range(date_range, site, expected_date_range) do + %{"metrics" => ["visitors", "events"], "date_range" => date_range} + |> check_success(site, %{ + metrics: [:visitors, :events], + date_range: expected_date_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "parsing empty map fails", %{site: site} do + %{} + |> check_error(site, "No valid metrics passed") + end + + describe "metrics validation" do + test "valid metrics passed", %{site: site} do + %{"metrics" => ["visitors", "events"], "date_range" => "all"} + |> check_success(site, %{ + metrics: [:visitors, :events], + date_range: @date_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "invalid metric passed", %{site: site} do + %{"metrics" => ["visitors", "event:name"], "date_range" => "all"} + |> check_error(site, "Unknown metric '\"event:name\"'") + end + + test "fuller list of metrics", %{site: site} do + %{ + "metrics" => [ + "time_on_page", + "visitors", + "pageviews", + "visits", + "events", + "bounce_rate", + "visit_duration" + ], + "date_range" => "all" + } + |> check_success(site, %{ + metrics: [ + :time_on_page, + :visitors, + :pageviews, + :visits, + :events, + :bounce_rate, + :visit_duration + ], + date_range: @date_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "same metric queried multiple times", %{site: site} do + %{"metrics" => ["events", "visitors", "visitors"], "date_range" => "all"} + |> check_error(site, ~r/Metrics cannot be queried multiple times/) + end + end + + describe "filters validation" do + for operation <- [:is, :is_not, :matches, :does_not_match] do + test "#{operation} filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [Atom.to_string(unquote(operation)), "event:name", ["foo"]] + ] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [ + [unquote(operation), "event:name", ["foo"]] + ], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "#{operation} filter with invalid clause", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [Atom.to_string(unquote(operation)), "event:name", "foo"] + ] + } + |> check_error(site, ~r/Invalid filter/) + end + end + + test "filtering by invalid operation", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["exists?", "event:name", ["foo"]] + ] + } + |> check_error(site, ~r/Unknown operator for filter/) + end + + test "filtering by custom properties", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:props:foobar", ["value"]] + ] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [ + [:is, "event:props:foobar", ["value"]] + ], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + for dimension <- Filters.event_props() do + if dimension != "goal" do + test "filtering by event:#{dimension} filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:#{unquote(dimension)}", ["foo"]] + ] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [ + [:is, "event:#{unquote(dimension)}", ["foo"]] + ], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + end + end + + for dimension <- Filters.visit_props() do + test "filtering by visit:#{dimension} filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "visit:#{unquote(dimension)}", ["foo"]] + ] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [ + [:is, "visit:#{unquote(dimension)}", ["foo"]] + ], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + end + + test "invalid event filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:device", ["foo"]] + ] + } + |> check_error(site, ~r/Invalid filter /) + end + + test "invalid visit filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "visit:name", ["foo"]] + ] + } + |> check_error(site, ~r/Invalid filter /) + end + + test "invalid filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => "foobar" + } + |> check_error(site, ~r/Invalid filters passed/) + end + end + + describe "include validation" do + test "setting include.imports", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{"imports" => true} + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: true, + preloaded_goals: [] + }) + end + + test "setting invalid imports value", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "include" => "foobar" + } + |> check_error(site, ~r/Invalid include passed/) + end + end + + describe "event:goal filter validation" do + test "valid filters", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, page_path: "/thank-you"}) + + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:goal", ["Signup", "Visit /thank-you"]] + ] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [ + [:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]] + ], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [{:page, "/thank-you"}, {:event, "Signup"}] + }) + end + + test "invalid event filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:goal", ["Signup"]] + ] + } + |> check_error(site, ~r/The goal `Signup` is not configured for this site/) + end + + test "invalid page filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:goal", ["Visit /thank-you"]] + ] + } + |> check_error(site, ~r/The goal `Visit \/thank-you` is not configured for this site/) + end + end + + describe "date range validation" do + test "parsing shortcut options", %{site: site} do + check_date_range("day", site, Date.range(~D[2021-05-05], ~D[2021-05-05])) + check_date_range("7d", site, Date.range(~D[2021-04-29], ~D[2021-05-05])) + check_date_range("30d", site, Date.range(~D[2021-04-05], ~D[2021-05-05])) + check_date_range("month", site, Date.range(~D[2021-05-01], ~D[2021-05-31])) + check_date_range("6mo", site, Date.range(~D[2020-12-01], ~D[2021-05-31])) + check_date_range("12mo", site, Date.range(~D[2020-06-01], ~D[2021-05-31])) + check_date_range("year", site, Date.range(~D[2021-01-01], ~D[2021-12-31])) + end + + test "parsing `all` with previous data", %{site: site} do + site = Map.put(site, :stats_start_date, ~D[2020-01-01]) + check_date_range("all", site, Date.range(~D[2020-01-01], ~D[2021-05-05])) + end + + test "parsing `all` with no previous data", %{site: site} do + site = Map.put(site, :stats_start_date, nil) + + check_date_range("all", site, Date.range(~D[2021-05-05], ~D[2021-05-05])) + end + + test "parsing custom date range", %{site: site} do + check_date_range( + ["2021-05-05", "2021-05-05"], + site, + Date.range(~D[2021-05-05], ~D[2021-05-05]) + ) + end + + test "parsing invalid custom date range", %{site: site} do + %{"date_range" => "foo", "metrics" => ["visitors"]} + |> check_error(site, ~r/Invalid date_range '\"foo\"'/) + + %{"date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]} + |> check_error(site, ~r/Invalid date_range /) + end + end + + describe "dimensions validation" do + for dimension <- Filters.event_props() do + test "event:#{dimension} dimension", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:#{unquote(dimension)}"] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [], + dimensions: ["event:#{unquote(dimension)}"], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + end + + for dimension <- Filters.visit_props() do + test "visit:#{dimension} dimension", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:#{unquote(dimension)}"] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [], + dimensions: ["visit:#{unquote(dimension)}"], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + end + + test "custom properties dimension", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:foobar"] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [], + dimensions: ["event:props:foobar"], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "invalid custom property dimension", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:"] + } + |> check_error(site, ~r/Invalid dimensions/) + end + + test "invalid dimension name passed", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visitors"] + } + |> check_error(site, ~r/Invalid dimensions/) + end + + test "invalid dimension", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => "foobar" + } + |> check_error(site, ~r/Invalid dimensions/) + end + + test "dimensions are not unique", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name", "event:name"] + } + |> check_error(site, ~r/Some dimensions are listed multiple times/) + end + end + + describe "order_by validation" do + test "ordering by metric", %{site: site} do + %{ + "metrics" => ["visitors", "events"], + "date_range" => "all", + "order_by" => [["events", "desc"], ["visitors", "asc"]] + } + |> check_success(site, %{ + metrics: [:visitors, :events], + date_range: @date_range, + filters: [], + dimensions: [], + order_by: [{:events, :desc}, {:visitors, :asc}], + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "ordering by dimension", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"], + "order_by" => [["event:name", "desc"]] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [], + dimensions: ["event:name"], + order_by: [{"event:name", :desc}], + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "ordering by invalid value", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "order_by" => [["visssss", "desc"]] + } + |> check_error(site, ~r/Invalid order_by entry/) + end + + test "ordering by not queried metric", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "order_by" => [["events", "desc"]] + } + |> check_error(site, ~r/Entry is not a queried metric or dimension/) + end + + test "ordering by not queried dimension", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "order_by" => [["event:name", "desc"]] + } + |> check_error(site, ~r/Entry is not a queried metric or dimension/) + end + end + + describe "custom props access" do + test "error if invalid filter", %{site: site, user: user} do + ep = + insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id) + + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "event:props:foobar", ["foo"]]] + } + |> check_error( + site, + ~r/The owner of this site does not have access to the custom properties feature/ + ) + end + + test "error if invalid dimension", %{site: site, user: user} do + ep = + insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id) + + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:foobar"] + } + |> check_error( + site, + ~r/The owner of this site does not have access to the custom properties feature/ + ) + end + end + + describe "conversion_rate metric" do + test "fails validation on its own", %{site: site} do + %{ + "metrics" => ["conversion_rate"], + "date_range" => "all" + } + |> check_error( + site, + ~r/Metric `conversion_rate` can only be queried with event:goal filters or dimensions/ + ) + end + + test "succeeds with event:goal filter", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + %{ + "metrics" => ["conversion_rate"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]] + } + |> check_success(site, %{ + metrics: [:conversion_rate], + date_range: @date_range, + filters: [[:is, "event:goal", [event: "Signup"]]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [event: "Signup"] + }) + end + + test "succeeds with event:goal dimension", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + %{ + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:goal"] + } + |> check_success(site, %{ + metrics: [:conversion_rate], + date_range: @date_range, + filters: [], + dimensions: ["event:goal"], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [event: "Signup"] + }) + end + end + + describe "views_per_visit metric" do + test "succeeds with normal filters", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + %{ + "metrics" => ["views_per_visit"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]] + } + |> check_success(site, %{ + metrics: [:views_per_visit], + date_range: @date_range, + filters: [[:is, "event:goal", [event: "Signup"]]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [event: "Signup"] + }) + end + + test "fails validation if event:page filter specified", %{site: site} do + %{ + "metrics" => ["views_per_visit"], + "date_range" => "all", + "filters" => [["is", "event:page", ["/"]]] + } + |> check_error( + site, + ~r/Metric `views_per_visit` cannot be queried with a filter on `event:page`/ + ) + end + + test "fails validation with dimensions", %{site: site} do + %{ + "metrics" => ["views_per_visit"], + "date_range" => "all", + "dimensions" => ["event:name"] + } + |> check_error( + site, + ~r/Metric `views_per_visit` cannot be queried with `dimensions`/ + ) + end + end + + describe "session metrics" do + test "single session metric succeeds", %{site: site} do + %{ + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"] + } + |> check_success(site, %{ + metrics: [:bounce_rate], + date_range: @date_range, + filters: [], + dimensions: ["visit:device"], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "fails if using session metric with event dimension", %{site: site} do + %{ + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:props:foo"] + } + |> check_error( + site, + "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" + ) + end + + test "does not fail if using session metric with event:page dimension", %{site: site} do + %{ + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:page"] + } + |> check_success(site, %{ + metrics: [:bounce_rate], + date_range: @date_range, + filters: [], + dimensions: ["event:page"], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "does not fail if using session metric with event filter", %{site: site} do + %{ + "metrics" => ["bounce_rate"], + "date_range" => "all", + "filters" => [["is", "event:props:foo", ["(none)"]]] + } + |> check_success(site, %{ + metrics: [:bounce_rate], + date_range: @date_range, + filters: [[:is, "event:props:foo", ["(none)"]]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + end +end diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index 629ea0344974..62c919319fab 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -194,14 +194,14 @@ defmodule Plausible.Stats.QueryTest do filters = Jason.encode!(%{"goal" => "Signup"}) q = Query.from(site, %{"period" => "6mo", "filters" => filters}) - assert q.filters == [[:is, "event:goal", {:event, "Signup"}]] + assert q.filters == [[:is, "event:goal", [{:event, "Signup"}]]] end test "parses source filter", %{site: site} do filters = Jason.encode!(%{"source" => "Twitter"}) q = Query.from(site, %{"period" => "6mo", "filters" => filters}) - assert q.filters == [[:is, "visit:source", "Twitter"]] + assert q.filters == [[:is, "visit:source", ["Twitter"]]] end end diff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exs index c7d85bbe7416..b216d02c7d74 100644 --- a/test/plausible/stats/table_decider_test.exs +++ b/test/plausible/stats/table_decider_test.exs @@ -5,62 +5,62 @@ defmodule Plausible.Stats.TableDeciderTest do import Plausible.Stats.TableDecider test "events_join_sessions? with experimental_reduced_joins disabled" do - assert not events_join_sessions?(make_query(false, %{})) - assert not events_join_sessions?(make_query(false, %{name: "pageview"})) - assert events_join_sessions?(make_query(false, %{source: "Google"})) - assert events_join_sessions?(make_query(false, %{entry_page: "/"})) - assert events_join_sessions?(make_query(false, %{exit_page: "/"})) + assert not events_join_sessions?(make_query(false, [])) + assert not events_join_sessions?(make_query(false, ["event:name"])) + assert events_join_sessions?(make_query(false, ["visit:source"])) + assert events_join_sessions?(make_query(false, ["visit:entry_page"])) + assert events_join_sessions?(make_query(false, ["visit:exit_page"])) end test "events_join_sessions? with experimental_reduced_joins enabled" do - assert not events_join_sessions?(make_query(true, %{})) - assert not events_join_sessions?(make_query(true, %{name: "pageview"})) - assert not events_join_sessions?(make_query(true, %{source: "Google"})) - assert events_join_sessions?(make_query(true, %{entry_page: "/"})) - assert events_join_sessions?(make_query(true, %{exit_page: "/"})) + assert not events_join_sessions?(make_query(true, [])) + assert not events_join_sessions?(make_query(true, ["event:name"])) + assert not events_join_sessions?(make_query(true, ["visit:source"])) + assert events_join_sessions?(make_query(true, ["visit:entry_page"])) + assert events_join_sessions?(make_query(true, ["visit:exit_page"])) end describe "partition_metrics" do test "with no metrics or filters" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([], query) == {[], [], []} end test "session-only metrics accordingly" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([:bounce_rate, :views_per_visit], query) == {[], [:bounce_rate, :views_per_visit], []} end test "event-only metrics accordingly" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([:total_revenue, :visitors], query) == {[:total_revenue, :visitors], [], []} end test "filters from both, event-only metrics" do - query = make_query(false, %{name: "pageview", source: "Google"}) + query = make_query(false, ["event:name", "visit:source"]) assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []} end test "filters from both, session-only metrics" do - query = make_query(false, %{name: "pageview", source: "Google"}) + query = make_query(false, ["event:name", "visit:source"]) assert partition_metrics([:bounce_rate], query) == {[], [:bounce_rate], []} end test "session filters but no session metrics" do - query = make_query(false, %{source: "Google"}) + query = make_query(false, ["visit:source"]) assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []} end test "sample_percent is added to both types of metrics" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([:total_revenue, :sample_percent], query) == {[:total_revenue, :sample_percent], [], []} @@ -73,14 +73,14 @@ defmodule Plausible.Stats.TableDeciderTest do end test "other metrics put in its own result" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) == - {[], [], [:time_on_page, :percentage, :total_visitors]} + {[], [:percentage], [:time_on_page, :total_visitors]} end test "raises if unknown metric" do - query = make_query(false, %{}) + query = make_query(false, []) assert_raise ArgumentError, fn -> partition_metrics([:foobar], query) @@ -90,7 +90,7 @@ defmodule Plausible.Stats.TableDeciderTest do describe "partition_metrics with experimental_reduced_joins enabled" do test "metrics that can be calculated on either when event-only metrics" do - query = make_query(true, %{}) + query = make_query(true, []) assert partition_metrics([:total_revenue, :visitors], query) == {[:total_revenue, :visitors], [], []} @@ -99,7 +99,7 @@ defmodule Plausible.Stats.TableDeciderTest do end test "metrics that can be calculated on either when session-only metrics" do - query = make_query(true, %{}) + query = make_query(true, []) assert partition_metrics([:bounce_rate, :visitors], query) == {[], [:bounce_rate, :visitors], []} @@ -108,57 +108,61 @@ defmodule Plausible.Stats.TableDeciderTest do {[], [:visit_duration, :visits], []} end - test "metrics that can be calculated on either are biased to sessions" do - query = make_query(true, %{}) + test "metrics that can be calculated on either are biased to events" do + query = make_query(true, []) assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) == - {[:total_revenue], [:bounce_rate, :visitors], []} + {[:total_revenue, :visitors], [:bounce_rate], []} end test "sample_percent is handled with either metrics" do - query = make_query(true, %{}) + query = make_query(true, []) assert partition_metrics([:visitors, :sample_percent], query) == {[], [:visitors, :sample_percent], []} end test "metric can be calculated on either, but filtering on events" do - query = make_query(true, %{name: "pageview"}) + query = make_query(true, ["event:name"]) assert partition_metrics([:visitors], query) == {[:visitors], [], []} end test "metric can be calculated on either, but filtering on events and sessions" do - query = make_query(true, %{name: "pageview", exit_page: "/"}) + query = make_query(true, ["event:name", "visit:exit_page"]) assert partition_metrics([:visitors], query) == {[], [:visitors], []} end test "metric can be calculated on either, filtering on either" do - query = make_query(true, %{source: "Google"}) + query = make_query(true, ["visit:source"]) assert partition_metrics([:visitors], query) == {[], [:visitors], []} end test "metric can be calculated on either, filtering on sessions" do - query = make_query(true, %{exit_page: "/"}) + query = make_query(true, ["visit:exit_page"]) assert partition_metrics([:visitors], query) == {[], [:visitors], []} end - test "breakdown value leans metric" do - query = make_query(true, %{}) + test "query dimensions lean metric" do + assert partition_metrics([:visitors], make_query(true, [], ["event:name"])) == + {[:visitors], [], []} - assert partition_metrics([:visitors], query, "event:name") == {[:visitors], [], []} - assert partition_metrics([:visitors], query, "visit:source") == {[], [:visitors], []} - assert partition_metrics([:visitors], query, "visit:exit_page") == {[], [:visitors], []} + assert partition_metrics([:visitors], make_query(true, [], ["visit:source"])) == + {[], [:visitors], []} + + assert partition_metrics([:visitors], make_query(true, [], ["visit:exit_page"])) == + {[], [:visitors], []} end end - defp make_query(experimental_reduced_joins?, filters) do + defp make_query(experimental_reduced_joins?, filter_keys, dimensions \\ []) do Query.from(build(:site), %{ "experimental_reduced_joins" => to_string(experimental_reduced_joins?), - "filters" => Jason.encode!(filters) + "filters" => Enum.map(filter_keys, fn filter_key -> ["is", filter_key, []] end), + "dimensions" => dimensions }) end end diff --git a/test/plausible_web/components/billing/notice_test.exs b/test/plausible_web/components/billing/notice_test.exs index 07eb69c38f2c..843172103157 100644 --- a/test/plausible_web/components/billing/notice_test.exs +++ b/test/plausible_web/components/billing/notice_test.exs @@ -1,6 +1,6 @@ defmodule PlausibleWeb.Components.Billing.NoticeTest do use Plausible.DataCase - import Phoenix.LiveViewTest + import Plausible.LiveViewTest, only: [render_component: 2] alias PlausibleWeb.Components.Billing.Notice test "premium_feature/1 does not render a notice when user is on trial" do diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv index 1cef2d8c42a7..4741db70e845 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv @@ -1,2 +1,2 @@ name,visitors,pageviews,bounce_rate,time_on_page -/some-other-page,1,1,,60.0 +/some-other-page,1,1,0,60.0 diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csv index e2990caa97ef..62b246c42004 100644 --- a/test/plausible_web/controllers/CSVs/30d/pages.csv +++ b/test/plausible_web/controllers/CSVs/30d/pages.csv @@ -1,4 +1,4 @@ name,visitors,pageviews,bounce_rate,time_on_page /,4,3,67, /signup,1,1,0,60.0 -/some-other-page,1,1,,60.0 +/some-other-page,1,1,0,60.0 diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csv index 8051585445aa..ed895e036f36 100644 --- a/test/plausible_web/controllers/CSVs/6m/pages.csv +++ b/test/plausible_web/controllers/CSVs/6m/pages.csv @@ -1,4 +1,4 @@ name,visitors,pageviews,bounce_rate,time_on_page /,5,4,75, /signup,1,1,0,60.0 -/some-other-page,1,1,,60.0 +/some-other-page,1,1,0,60.0 diff --git a/test/plausible_web/controllers/admin_controller_test.exs b/test/plausible_web/controllers/admin_controller_test.exs index 90321bb1e3ac..2985320c0861 100644 --- a/test/plausible_web/controllers/admin_controller_test.exs +++ b/test/plausible_web/controllers/admin_controller_test.exs @@ -11,6 +11,53 @@ defmodule PlausibleWeb.AdminControllerTest do conn = get(conn, "/crm/auth/user/1/usage") assert response(conn, 403) == "Not allowed" end + + @tag :ee_only + test "returns usage data as a standalone page", %{conn: conn, user: user} do + patch_env(:super_admin_user_ids, [user.id]) + conn = get(conn, "/crm/auth/user/#{user.id}/usage") + assert response(conn, 200) =~ " "2"}) + page1_html = html_response(conn1, 200) + + assert page1_html =~ s1.domain + assert page1_html =~ s2.domain + refute page1_html =~ s3.domain + + conn2 = get(conn, "/crm/sites/site", %{"page" => "2", "limit" => "2"}) + page2_html = html_response(conn2, 200) + + refute page2_html =~ s1.domain + refute page2_html =~ s2.domain + assert page2_html =~ s3.domain + end end describe "POST /crm/sites/site/:site_id" do @@ -48,4 +95,59 @@ defmodule PlausibleWeb.AdminControllerTest do assert site.stats_start_date == nil end end + + describe "GET /crm/billing/user/:user_id/current_plan" do + setup [:create_user, :log_in] + + @tag :ee_only + test "returns 403 if the logged in user is not a super admin", %{conn: conn} do + conn = get(conn, "/crm/billing/user/0/current_plan") + assert response(conn, 403) == "Not allowed" + end + + @tag :ee_only + test "returns empty state for non-existent user", %{conn: conn, user: user} do + patch_env(:super_admin_user_ids, [user.id]) + + conn = get(conn, "/crm/billing/user/0/current_plan") + assert json_response(conn, 200) == %{"features" => []} + end + + @tag :ee_only + test "returns empty state for user without subscription", %{conn: conn, user: user} do + patch_env(:super_admin_user_ids, [user.id]) + + conn = get(conn, "/crm/billing/user/#{user.id}/current_plan") + assert json_response(conn, 200) == %{"features" => []} + end + + @tag :ee_only + test "returns empty state for user with subscription with non-existent paddle plan ID", %{ + conn: conn, + user: user + } do + patch_env(:super_admin_user_ids, [user.id]) + + insert(:subscription, user: user) + + conn = get(conn, "/crm/billing/user/#{user.id}/current_plan") + assert json_response(conn, 200) == %{"features" => []} + end + + @tag :ee_only + test "returns plan data for user with subscription", %{conn: conn, user: user} do + patch_env(:super_admin_user_ids, [user.id]) + + insert(:subscription, user: user, paddle_plan_id: "857104") + + conn = get(conn, "/crm/billing/user/#{user.id}/current_plan") + + assert json_response(conn, 200) == %{ + "features" => ["goals"], + "monthly_pageview_limit" => 10_000_000, + "site_limit" => 10, + "team_member_limit" => 3 + } + end + end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs index 68cfefd85968..ca540b427d33 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs @@ -1746,4 +1746,245 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do assert json_response(conn, 200)["results"] == %{"conversion_rate" => %{"value" => 0}} end end + + describe "with json filters" do + test "filtering by exact string", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en*"), + build(:pageview, pathname: "/en*/page1"), + build(:pageview, pathname: "/en*/page2"), + build(:pageview, pathname: "/ena/page2"), + build(:pageview, pathname: "/pll/page1") + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "metrics" => "visitors", + "filters" => [ + ["is", "event:page", ["/en*"]] + ] + }) + + assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 1}} + end + + test "filtering by goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/blog/post-1"), + build(:pageview, pathname: "/blog/post-2", user_id: @user_id), + build(:pageview, pathname: "/blog", user_id: @user_id), + build(:pageview, pathname: "/") + ]) + + insert(:goal, %{site: site, page_path: "/blog"}) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "metrics" => "visitors,pageviews", + "filters" => [["is", "event:goal", ["Visit /blog"]]] + }) + + assert json_response(conn, 200)["results"] == %{ + "visitors" => %{"value" => 1}, + "pageviews" => %{"value" => 1} + } + end + + test "filtering by wildcard goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/blog/post-1"), + build(:pageview, pathname: "/blog/post-2", user_id: @user_id), + build(:pageview, pathname: "/blog", user_id: @user_id), + build(:pageview, pathname: "/") + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "metrics" => "visitors,pageviews", + "filters" => [["is", "event:goal", ["Visit /blog**"]]] + }) + + assert json_response(conn, 200)["results"] == %{ + "visitors" => %{"value" => 2}, + "pageviews" => %{"value" => 3} + } + end + + test "contains", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en*"), + build(:pageview, pathname: "/en*/page1"), + build(:pageview, pathname: "/en*/page2"), + build(:pageview, pathname: "/ena/page2"), + build(:pageview, pathname: "/pll/page1") + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "metrics" => "visitors", + "filters" => [ + ["contains", "event:page", ["/en*"]] + ] + }) + + assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}} + end + + test "does not contain", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en*"), + build(:pageview, pathname: "/en*/page1"), + build(:pageview, pathname: "/en*/page2"), + build(:pageview, pathname: "/ena/page2"), + build(:pageview, pathname: "/pll/page1") + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "metrics" => "visitors", + "filters" => [ + ["does_not_contain", "event:page", ["/en*"]] + ] + }) + + assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 2}} + end + + test "matches custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["large-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-2"] + ) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "metrics" => "visitors", + "filters" => [ + ["matches", "event:props:tier", ["small*"]] + ] + }) + + assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}} + end + + test "does_not_match custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["large-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-2"] + ) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "metrics" => "visitors", + "filters" => [ + ["does_not_match", "event:props:tier", ["small*"]] + ] + }) + + assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 1}} + end + + test "contains custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["large-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-2"] + ) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "metrics" => "visitors", + "filters" => [ + ["contains", "event:props:tier", ["small"]] + ] + }) + + assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}} + end + + test "does_not_contain custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["large-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-1"] + ), + build(:pageview, + "meta.key": ["tier"], + "meta.value": ["small-2"] + ) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "metrics" => "visitors", + "filters" => [ + ["does_not_contain", "event:props:tier", ["small"]] + ] + }) + + assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 1}} + end + end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/auth_test.exs b/test/plausible_web/controllers/api/external_stats_controller/auth_test.exs index 3d66538c0646..9107fd381826 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/auth_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/auth_test.exs @@ -150,6 +150,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AuthTest do }) end + @tag :ee_only test "returns HTTP 402 when user is on a growth plan", %{ conn: conn, user: user, diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs index 8be42e318717..d6b9a305b684 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs @@ -890,8 +890,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert json_response(conn, 200) == %{ "results" => [ %{"page" => "/", "pageviews" => 2}, - %{"page" => "/plausible.io", "pageviews" => 1}, - %{"page" => "/include-me", "pageviews" => 1} + %{"page" => "/include-me", "pageviews" => 1}, + %{"page" => "/plausible.io", "pageviews" => 1} ] } end @@ -1023,7 +1023,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do "period" => "day", "date" => "2021-01-01", "property" => "visit:exit_page", - "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration,events", + "metrics" => "visitors,visits,bounce_rate,visit_duration,events,pageviews", "with_imported" => "true" }) @@ -1031,21 +1031,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do "results" => [ %{ "bounce_rate" => 0.0, - "events" => 7, "exit_page" => "/b", - "pageviews" => 7, "visit_duration" => 150.0, "visitors" => 3, - "visits" => 4 + "visits" => 4, + "events" => 7, + "pageviews" => 7 }, %{ "bounce_rate" => 100.0, - "events" => 1, "exit_page" => "/a", - "pageviews" => 1, "visit_duration" => 0.0, "visitors" => 1, - "visits" => 1 + "visits" => 1, + "events" => 1, + "pageviews" => 1 } ] } @@ -2176,8 +2176,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert json_response(conn, 200) == %{ "results" => [ - %{"page" => "/plausible.io", "bounce_rate" => 100}, - %{"page" => "/important-page", "bounce_rate" => 100} + %{"page" => "/important-page", "bounce_rate" => 100}, + %{"page" => "/plausible.io", "bounce_rate" => 100} ] } end @@ -2596,14 +2596,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert json_response(conn, 200) == %{ "results" => [ - %{ - "page" => "/B", - "time_on_page" => 90.0 - }, %{ "page" => "/A", "time_on_page" => 60.0 }, + %{ + "page" => "/B", + "time_on_page" => 90.0 + }, %{ "page" => "/C", "time_on_page" => nil @@ -3045,13 +3045,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert json_response(conn, 200) == %{ "results" => [ - %{ - "entry_page" => "/entry-page-1", - "bounce_rate" => 0 - }, %{ "entry_page" => "/entry-page-2", "bounce_rate" => 100 + }, + %{ + "entry_page" => "/entry-page-1", + "bounce_rate" => 0 } ] } @@ -3146,6 +3146,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do }, %{ "page" => "/plausible.io", + # Breaks for event:page breakdown since visitors is calculated based on entry_page :/ "visitors" => 2, "bounce_rate" => 100, "visit_duration" => 0, @@ -3290,8 +3291,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser") assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device") - assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page") - assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page") assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country") assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os") assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page") diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs new file mode 100644 index 000000000000..331687e1f0ae --- /dev/null +++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs @@ -0,0 +1,3992 @@ +defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do + use PlausibleWeb.ConnCase + alias Plausible.Billing.Feature + + @user_id 1231 + + setup [:create_user, :create_new_site, :create_api_key, :use_api_key] + + describe "feature access" do + test "cannot break down by a custom prop without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:author"] + }) + + assert json_response(conn, 400)["error"] == + "The owner of this site does not have access to the custom properties feature" + end + + test "can break down by an internal prop key without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:path"] + }) + + assert json_response(conn, 200)["results"] + end + + test "cannot filter by a custom prop without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = + insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [["is", "event:props:author", ["Uku"]]] + }) + + assert json_response(conn, 400)["error"] == + "The owner of this site does not have access to the custom properties feature" + end + + test "can filter by an internal prop key without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [["is", "event:props:url", ["whatever"]]] + }) + + assert json_response(conn, 200)["results"] + end + end + + describe "param validation" do + test "does not allow querying conversion_rate without a goal filter", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [["is", "event:props:author", ["Uku"]]] + }) + + assert json_response(conn, 400)["error"] == + "Metric `conversion_rate` can only be queried with event:goal filters or dimensions" + end + + test "validates that dimensions are valid", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["badproperty"] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid dimensions" + end + + test "empty custom property is invalid", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:"] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid dimensions" + end + + test "validates that correct date range is used", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "bad_period", + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid date_range" + end + + test "fails when an invalid metric is provided", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "baa"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'" + end + + test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400)["error"] =~ + "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" + end + + test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:props:url"] + }) + + assert json_response(conn, 400)["error"] =~ + "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" + end + + test "validates that metric views_per_visit cannot be used with event:page filter", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["views_per_visit"], + "filters" => [["is", "event:page", ["/something"]]] + }) + + assert json_response(conn, 400) == %{ + "error" => + "Metric `views_per_visit` cannot be queried with a filter on `event:page`" + } + end + + test "validates that metric views_per_visit cannot be used together with dimensions", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["views_per_visit"], + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400) == %{ + "error" => "Metric `views_per_visit` cannot be queried with `dimensions`" + } + end + + test "validates a metric can't be asked multiple times", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["views_per_visit", "visitors", "visitors"] + }) + + assert json_response(conn, 400) == %{ + "error" => "Metrics cannot be queried multiple times" + } + end + end + + test "aggregates a single metric", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => "all" + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}] + end + + test "aggregate views_per_visit rounds to two decimal places", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["views_per_visit"], + "date_range" => "all" + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [1.67], "dimensions" => []}] + end + + test "aggregates all metrics in a single query", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => [ + "pageviews", + "visits", + "views_per_visit", + "visitors", + "bounce_rate", + "visit_duration" + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [3, 2, 1.5, 2, 50, 750], "dimensions" => []} + ] + end + + describe "aggregation with filters" do + test "can filter by source", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:source", ["Google"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by no source/referrer", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:source", ["Direct / None"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by referrer", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer: "https://facebook.com", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:referrer", ["https://facebook.com"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "wildcard referrer filter with special regex characters", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, referrer: "https://a.com"), + build(:pageview, referrer: "https://a.com"), + build(:pageview, referrer: "https://ab.com") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["matches", "visit:referrer", ["**a.com**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] + end + + test "can filter by utm_medium", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_medium: "social", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:utm_medium", ["social"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by utm_source", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_source: "Twitter", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:utm_source", ["Twitter"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by utm_campaign", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_campaign: "profile", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:utm_campaign", ["profile"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by device type", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + screen_size: "Desktop", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:device", ["Desktop"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by browser", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:browser", ["Chrome"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by browser version", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + browser_version: "56", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:browser_version", ["56"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by operating system", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + operating_system: "Mac", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:os", ["Mac"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by operating system version", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + operating_system_version: "10.5", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:os_version", ["10.5"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by country", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + country_code: "EE", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:country", ["EE"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by page", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + pathname: "/blogpost", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, + pathname: "/blogpost", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [ + ["is", "event:page", ["/blogpost"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 2, 100, 750], "dimensions" => []} + ] + end + + test "can filter by hostname", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + hostname: "one.example.com", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + hostname: "example.com", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [ + ["matches", "event:hostname", ["*.example.com", "example.com"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 2, 100, 0], "dimensions" => []} + ] + end + + test "filtering by event:name", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["is", "event:name", ["Signup"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 0], "dimensions" => []}] + end + + test "filtering by a custom event goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "NotConfigured", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by a revenue goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + insert(:goal, site: site, currency: :USD, event_name: "Purchase") + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["is", "event:goal", ["Purchase"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by a simple pageview goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/register", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/register", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + pathname: "/register", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + pathname: "/", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + insert(:goal, %{site: site, page_path: "/register"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["is", "event:goal", ["Visit /register"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by a wildcard pageview goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/blog/post-1"), + build(:pageview, pathname: "/blog/post-2", user_id: @user_id), + build(:pageview, pathname: "/blog", user_id: @user_id), + build(:pageview, pathname: "/") + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["matches", "event:goal", ["Visit /blog**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by multiple custom event goals", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "Signup"), + build(:event, name: "Purchase", user_id: @user_id), + build(:event, name: "Purchase", user_id: @user_id), + build(:pageview) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, event_name: "Purchase"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["is", "event:goal", ["Signup", "Purchase"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by multiple mixed goals", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/account/register"), + build(:pageview, pathname: "/register", user_id: @user_id), + build(:event, name: "Signup", user_id: @user_id), + build(:pageview) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, page_path: "/**register"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events", "pageviews"], + "filters" => [ + ["matches", "event:goal", ["Signup", "Visit /**register"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 3, 2], "dimensions" => []} + ] + end + + test "combining filters", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/blogpost", + country_code: "EE", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, + pathname: "/blogpost", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [ + ["is", "event:page", ["/blogpost"]], + ["is", "visit:country", ["EE"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [1, 1, 0, 1500], "dimensions" => []} + ] + end + + test "wildcard page filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/pl/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["matches", "event:page", ["/en/**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] + end + + test "negated wildcard page filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/pl/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["does_not_match", "event:page", ["/en/**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}] + end + + test "wildcard and member filter combined", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/pl/page1"), + build(:pageview, pathname: "/ee/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["matches", "event:page", ["/en/**", "/pl/**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}] + end + + test "can escape pipe character in member + wildcard filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/blog/post|1"), + build(:pageview, pathname: "/otherpost|1"), + build(:pageview, pathname: "/blog/post|2"), + build(:pageview, pathname: "/something-else") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["matches", "event:page", ["**post\\|1", "/something-else"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}] + end + + test "handles filtering by visit country", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, country_code: "EE"), + build(:pageview, country_code: "EE"), + build(:pageview, country_code: "EE") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews"], + "filters" => [["is", "visit:country", ["EE"]]] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}] + end + end + + describe "aggregation with imported data" do + setup :create_site_import + + test "does not count imported stats unless specified", %{ + conn: conn, + site: site, + site_import: site_import + } do + populate_stats(site, site_import.id, [ + build(:imported_visitors, date: ~D[2023-01-01]), + build(:pageview, timestamp: ~N[2023-01-01 00:00:00]) + ]) + + query_params = %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews"] + } + + conn1 = post(conn, "/api/v2/query", query_params) + + assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}] + + conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true})) + + assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] + refute json_response(conn2, 200)["meta"]["warning"] + end + end + + describe "timeseries" do + test "shows hourly data for a certain date", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:10:00]), + build(:pageview, timestamp: ~N[2021-01-01 23:59:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "pageviews", "visits", "visit_duration", "bounce_rate"], + "date_range" => ["2021-01-01", "2021-01-01"], + "dimensions" => ["time:hour"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-01T00:00:00Z"], "metrics" => [1, 2, 1, 600, 0]}, + %{"dimensions" => ["2021-01-01T23:00:00Z"], "metrics" => [1, 1, 1, 0, 100]} + ] + end + + test "shows last 7 days of visitors", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-07 23:59:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2021-01-01", "2021-01-07"], + "dimensions" => ["time"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-07"], "metrics" => [1]} + ] + end + + test "shows last 6 months of visitors", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2020-08-13 00:00:00]), + build(:pageview, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2020-07-01", "2021-01-31"], + "dimensions" => ["time"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2020-08-01"], "metrics" => [1]}, + %{"dimensions" => ["2020-12-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [2]} + ] + end + + test "shows last 12 months of visitors", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2020-02-01 00:00:00]), + build(:pageview, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2020-01-01", "2021-01-01"], + "dimensions" => ["time"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2020-02-01"], "metrics" => [1]}, + %{"dimensions" => ["2020-12-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [2]} + ] + end + + test "shows last 12 months of visitors with interval daily", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2020-02-01 00:00:00]), + build(:pageview, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2020-01-01", "2021-01-07"], + "dimensions" => ["time:day"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2020-02-01"], "metrics" => [1]}, + %{"dimensions" => ["2020-12-31"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [2]} + ] + end + + test "shows a custom range with daily interval", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2021-01-01", "2021-01-02"], + "dimensions" => ["time:day"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-02"], "metrics" => [1]} + ] + end + + test "shows a custom range with monthly interval", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:05:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "date_range" => ["2020-12-01", "2021-01-02"], + "dimensions" => ["time:month"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2020-12-01"], "metrics" => [2, 1, 0, 300]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [2, 2, 100, 0]} + ] + end + + test "timeseries with explicit order_by", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-03 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-03 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-04 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-04 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => ["2020-12-01", "2021-01-04"], + "dimensions" => ["time"], + "order_by" => [["pageviews", "desc"], ["time", "asc"]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-02"], "metrics" => [3]}, + %{"dimensions" => ["2021-01-03"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-04"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [1]} + ] + end + end + + test "breakdown by visit:source", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + referrer_source: "", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [2]}, + %{"dimensions" => ["Direct / None"], "metrics" => [1]} + ] + end + + test "breakdown by visit:country", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, country_code: "US", timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:country"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["EE"], "metrics" => [2]}, + %{"dimensions" => ["US"], "metrics" => [1]} + ] + end + + test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do + site_import = + insert(:site_import, + site: site, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]), + build(:imported_sources, + referrer: "site.com", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 1, + visit_duration: 120 + ), + build(:imported_sources, + referrer: "site.com/2", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 2, + visit_duration: 0 + ), + build(:imported_sources, + date: ~D[2021-01-01], + visitors: 10, + visits: 11, + pageviews: 50, + bounces: 0, + visit_duration: 1100 + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], + "date_range" => "all", + "dimensions" => ["visit:referrer"], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]}, + %{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]}, + %{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]}, + %{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]} + ] + end + + for {dimension, attr} <- [ + {"visit:utm_campaign", :utm_campaign}, + {"visit:utm_source", :utm_source}, + {"visit:utm_term", :utm_term}, + {"visit:utm_content", :utm_content} + ] do + test "breakdown by #{dimension} when filtered by hostname", %{conn: conn, site: site} do + populate_stats(site, [ + # session starts at two.example.com with utm_param=ad + build( + :pageview, + [ + {unquote(attr), "ad"}, + {:user_id, @user_id}, + {:hostname, "two.example.com"}, + {:timestamp, ~N[2021-01-01 00:00:00]} + ] + ), + # session continues on one.example.com without any utm_params + build( + :pageview, + [ + {:user_id, @user_id}, + {:hostname, "one.example.com"}, + {:timestamp, ~N[2021-01-01 00:15:00]} + ] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "event:hostname", ["one.example.com"]]], + "dimensions" => [unquote(dimension)] + }) + + # nobody landed on one.example.com from utm_param=ad + assert json_response(conn, 200)["results"] == [] + end + end + + for {dimension, column, value1, value2, blank_value} <- [ + {"visit:source", :referrer_source, "Google", "Twitter", "Direct / None"}, + {"visit:referrer", :referrer, "example.com", "google.com", "Direct / None"}, + {"visit:utm_medium", :utm_medium, "Search", "social", "(not set)"}, + {"visit:utm_source", :utm_source, "Google", "Bing", "(not set)"}, + {"visit:utm_campaign", :utm_campaign, "ads", "profile", "(not set)"}, + {"visit:utm_content", :utm_content, "Content1", "blog2", "(not set)"}, + {"visit:utm_term", :utm_term, "Term1", "favicon", "(not set)"}, + {"visit:os", :operating_system, "Mac", "Windows", "(not set)"}, + {"visit:browser", :browser, "Chrome", "Safari", "(not set)"}, + {"visit:device", :screen_size, "Mobile", "Desktop", "(not set)"} + ] do + test "simple breakdown by #{dimension}", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, [ + {unquote(column), unquote(value1)}, + {:timestamp, ~N[2021-01-01 00:00:00]} + ]), + build(:pageview, [ + {unquote(column), unquote(value1)}, + {:timestamp, ~N[2021-01-01 00:25:00]} + ]), + build(:pageview, [ + {unquote(column), unquote(value1)}, + {:timestamp, ~N[2021-01-01 00:55:00]} + ]), + build(:pageview, [ + {unquote(column), unquote(value2)}, + {:timestamp, ~N[2021-01-01 01:00:00]} + ]), + build(:pageview, [ + {unquote(column), unquote(value2)}, + {:timestamp, ~N[2021-01-01 01:25:00]} + ]), + build(:pageview, [ + {unquote(column), ""}, + {:timestamp, ~N[2021-01-01 00:00:00]} + ]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "percentage"], + "date_range" => ["2021-01-01", "2021-01-01"], + "dimensions" => [unquote(dimension)] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => [unquote(value1)], "metrics" => [3, 50]}, + %{"dimensions" => [unquote(value2)], "metrics" => [2, 33.3]}, + %{"dimensions" => [unquote(blank_value)], "metrics" => [1, 16.7]} + ] + end + end + + test "breakdown by visit:os and visit:os_version", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system_version: "14"), + build(:pageview, + operating_system: "Windows", + operating_system_version: "11" + ), + build(:pageview, + operating_system: "Windows", + operating_system_version: "11" + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:os", "visit:os_version"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Mac", "14"], "metrics" => [3]}, + %{"dimensions" => ["Windows", "11"], "metrics" => [2]}, + %{"dimensions" => ["(not set)", "14"], "metrics" => [1]} + ] + end + + test "breakdown by visit:browser and visit:browser_version", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, browser: "Chrome", browser_version: "14"), + build(:pageview, browser: "Chrome", browser_version: "14"), + build(:pageview, browser: "Chrome", browser_version: "14"), + build(:pageview, browser_version: "14"), + build(:pageview, + browser: "Firefox", + browser_version: "11" + ), + build(:pageview, + browser: "Firefox", + browser_version: "11" + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser", "visit:browser_version"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Chrome", "14"], "metrics" => [3]}, + %{"dimensions" => ["Firefox", "11"], "metrics" => [2]}, + %{"dimensions" => ["(not set)", "14"], "metrics" => [1]} + ] + end + + test "explicit order_by", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, operating_system: "Windows", browser: "Chrome"), + build(:pageview, operating_system: "Windows", browser: "Firefox"), + build(:pageview, operating_system: "Linux", browser: "Firefox"), + build(:pageview, operating_system: "Linux", browser: "Firefox"), + build(:pageview, operating_system: "Mac", browser: "Chrome") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:os", "visit:browser"], + "order_by" => [["visitors", "asc"], ["visit:browser", "desc"], ["visit:os", "asc"]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Windows", "Firefox"], "metrics" => [1]}, + %{"dimensions" => ["Mac", "Chrome"], "metrics" => [1]}, + %{"dimensions" => ["Windows", "Chrome"], "metrics" => [1]}, + %{"dimensions" => ["Linux", "Firefox"], "metrics" => [2]} + ] + end + + test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do + site_import = + insert(:site_import, + site: site, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]), + build(:imported_sources, + utm_source: "SomeUTMSource", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 1, + visit_duration: 120 + ), + build(:imported_sources, + utm_source: "SomeUTMSource-2", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 2, + visit_duration: 0 + ), + build(:imported_sources, + date: ~D[2021-01-01], + visitors: 10, + visits: 11, + pageviews: 50, + bounces: 0, + visit_duration: 1100 + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], + "date_range" => "all", + "dimensions" => ["visit:utm_source"], + "include" => %{"imports" => true} + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]}, + %{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]}, + %{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]} + ] + end + + test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed", + %{conn: conn, site: site} do + site_import = + insert(:site_import, + site: site, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:imported_pages, + page: "/skip-me", + date: ~D[2021-01-01], + visitors: 1, + pageviews: 0 + ), + build(:imported_pages, + page: "/include-me", + date: ~D[2021-01-01], + visitors: 1, + pageviews: 1 + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => "all", + "dimensions" => ["event:page"], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/"], "metrics" => [2]}, + %{"dimensions" => ["/plausible.io"], "metrics" => [1]}, + %{"dimensions" => ["/include-me"], "metrics" => [1]} + ] + end + + test "breakdown by event:page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/"], "metrics" => [2]}, + %{"dimensions" => ["/plausible.io"], "metrics" => [1]} + ] + end + + test "attempting to breakdown by event:hostname returns an error", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, hostname: "a.example.com"), + build(:pageview, hostname: "a.example.com"), + build(:pageview, hostname: "a.example.com"), + build(:pageview, hostname: "b.example.com") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews"], + "dimensions" => ["event:hostname"], + "with_imported" => "true" + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["a.example.com"], "metrics" => [3]}, + %{"dimensions" => ["b.example.com"], "metrics" => [1]} + ] + end + + describe "breakdown by visit:exit_page" do + setup %{site: site} do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + pathname: "/a", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + pathname: "/a", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + user_id: @user_id, + pathname: "/b", + timestamp: ~N[2021-01-01 00:35:00] + ), + build(:imported_exit_pages, + exit_page: "/b", + exits: 3, + visitors: 2, + pageviews: 5, + date: ~D[2021-01-01] + ) + ]) + end + + test "can query with visit:exit_page dimension", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visits"], + "date_range" => "all", + "dimensions" => ["visit:exit_page"], + "include" => %{"imports" => true} + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/b"], "metrics" => [4]}, + %{"dimensions" => ["/a"], "metrics" => [1]} + ] + end + end + + describe "custom events" do + test "can breakdown by event:name", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Signup"], "metrics" => [2]}, + %{"dimensions" => ["pageview"], "metrics" => [1]} + ] + end + + test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "404", + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/non-existing", + timestamp: ~N[2021-01-01 00:00:01] + ), + build(:pageview, + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:02] + ), + build(:event, + name: "404", + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:02] + ), + build(:pageview, + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:03] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["pageview"], "metrics" => [2, 4]}, + %{"dimensions" => ["404"], "metrics" => [1, 2]} + ] + end + + test "can breakdown by event:name while filtering for something", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + pathname: "/pageA", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageA", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageA", + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageB", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/pageA", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"], + "filters" => [ + ["is", "event:page", ["/pageA"]], + ["is", "visit:browser", ["Chrome"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Signup"], "metrics" => [2]}, + %{"dimensions" => ["pageview"], "metrics" => [1]} + ] + end + + test "can breakdown by a visit:property when filtering by event:name", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Twitter", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "event:name", ["Signup"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [1]} + ] + end + + test "can breakdown by event:name when filtering by event:page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/pageA", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/pageA", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + pathname: "/pageA", + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/pageB", + referrer_source: "Twitter", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"], + "filters" => [ + ["matches", "event:page", ["/pageA"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["pageview"], "metrics" => [2]}, + %{"dimensions" => ["Signup"], "metrics" => [1]} + ] + end + + test "can breakdown by event:page when filtering by event:name", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + pathname: "/pageA", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageA", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageB", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/pageB", + referrer_source: "Twitter", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "event:name", ["Signup"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/pageA"], "metrics" => [2]}, + %{"dimensions" => ["/pageB"], "metrics" => [1]} + ] + end + + test "can filter event:page with a wildcard", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/pl/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["matches", "event:page", ["/en/**"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/en/page2"], "metrics" => [2]}, + %{"dimensions" => ["/en/page1"], "metrics" => [1]} + ] + end + + test "can filter event:hostname with a wildcard", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, hostname: "alice.example.com", pathname: "/a"), + build(:pageview, hostname: "anna.example.com", pathname: "/a"), + build(:pageview, hostname: "adam.example.com", pathname: "/a"), + build(:pageview, hostname: "bob.example.com", pathname: "/b") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["matches", "event:hostname", ["a*.example.com"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/a"], "metrics" => [3]} + ] + end + + test "breakdown by custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["personal"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Some other event", + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:name", ["Purchase"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["business"], "metrics" => [2]}, + %{"dimensions" => ["personal"], "metrics" => [1]} + ] + end + + test "breakdown by custom event property, with pageviews metric", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["package"], + "meta.value": ["personal"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["business"], "metrics" => [2]}, + %{"dimensions" => ["personal"], "metrics" => [1]} + ] + end + + test "breakdown by custom event property, with (none)", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["16"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["16"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["16"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["14"], + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["14"], + timestamp: ~N[2021-01-01 00:26:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:cost"], + "filters" => [["is", "event:name", ["Purchase"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["16"], "metrics" => [3]}, + %{"dimensions" => ["14"], "metrics" => [2]}, + %{"dimensions" => ["(none)"], "metrics" => [1]} + ] + end + end + + describe "breakdown by event:goal" do + test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + insert(:goal, %{site: site, event_name: "Purchase"}) + insert(:goal, %{site: site, page_path: "/test"}) + + populate_stats(site, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + pathname: "/test" + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Purchase"], "metrics" => [2]}, + %{"dimensions" => ["Visit /test"], "metrics" => [1]} + ] + end + + test "returns pageview goals containing wildcards", %{conn: conn, site: site} do + insert(:goal, %{site: site, page_path: "/**/post"}) + insert(:goal, %{site: site, page_path: "/blog**"}) + + populate_stats(site, [ + build(:pageview, pathname: "/blog", user_id: @user_id), + build(:pageview, pathname: "/blog/post-1", user_id: @user_id), + build(:pageview, pathname: "/blog/post-2", user_id: @user_id), + build(:pageview, pathname: "/blog/something/post"), + build(:pageview, pathname: "/different/page/post") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "dimensions" => ["event:goal"], + "order_by" => [["pageviews", "desc"]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]}, + %{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]} + ] + end + + test "does not return goals that are not configured for the site", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/register"), + build(:event, name: "Signup") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [] + end + end + + test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:goal", ["Register"]] + ] + }) + + assert %{"error" => msg} = json_response(conn, 400) + assert msg =~ "The goal `Register` is not configured for this site. Find out how" + end + + test "validates that filters are valid", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "badproperty", ["bar"]] + ] + }) + + assert %{"error" => msg} = json_response(conn, 400) + assert msg =~ "Invalid filter" + end + + test "event:page filter for breakdown by session props", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/ignore", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + browser: "Safari", + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:page", ["/plausible.io"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Chrome"], "metrics" => [2]}, + %{"dimensions" => ["Safari"], "metrics" => [1]} + ] + end + + test "event:page filter shows sources of sessions that have visited that page", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + pathname: "/", + referrer_source: "Twitter", + utm_medium: "Twitter", + utm_source: "Twitter", + utm_campaign: "Twitter", + user_id: @user_id + ), + build(:pageview, + pathname: "/plausible.io", + user_id: @user_id + ), + build(:pageview, + pathname: "/plausible.io", + referrer_source: "Google", + utm_medium: "Google", + utm_source: "Google", + utm_campaign: "Google" + ), + build(:pageview, + pathname: "/plausible.io", + referrer_source: "Google", + utm_medium: "Google", + utm_source: "Google", + utm_campaign: "Google" + ) + ]) + + for dimension <- [ + "visit:source", + "visit:utm_medium", + "visit:utm_source", + "visit:utm_campaign" + ] do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => [dimension], + "filters" => [ + ["is", "event:page", ["/plausible.io"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [2]}, + %{"dimensions" => ["Twitter"], "metrics" => [1]} + ] + end + end + + test "top sources for a custom goal and filtered by hostname", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + hostname: "blog.example.com", + referrer_source: "Facebook", + user_id: @user_id + ), + build(:pageview, + hostname: "app.example.com", + pathname: "/register", + user_id: @user_id + ), + build(:event, + name: "Signup", + hostname: "app.example.com", + pathname: "/register", + user_id: @user_id + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "event:hostname", ["app.example.com"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [] + end + + test "top sources for a custom goal and filtered by hostname (2)", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + hostname: "app.example.com", + referrer_source: "Facebook", + pathname: "/register", + user_id: @user_id + ), + build(:event, + name: "Signup", + hostname: "app.example.com", + pathname: "/register", + user_id: @user_id + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "event:hostname", ["app.example.com"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Facebook"], "metrics" => [1]} + ] + end + + test "event:page filter is interpreted as entry_page filter only for bounce_rate", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + user_id: @user_id, + pathname: "/ignore", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + user_id: @user_id, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + browser: "Chrome", + user_id: 456, + pathname: "/important-page", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + user_id: 456, + pathname: "/", + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + browser: "Chrome", + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:01:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "bounce_rate"], + "filters" => [["is", "event:page", ["/plausible.io", "/important-page"]]], + "dimensions" => ["visit:browser"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Chrome"], "metrics" => [3, 50]} + ] + end + + test "event:goal pageview filter for breakdown by visit source", %{conn: conn, site: site} do + insert(:goal, %{site: site, page_path: "/plausible.io"}) + + populate_stats(site, [ + build(:pageview, + referrer_source: "Bing", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Google", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "event:goal", ["Visit /plausible.io"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [1]} + ] + end + + test "event:goal custom event filter for breakdown by visit source", %{conn: conn, site: site} do + insert(:goal, %{site: site, event_name: "Register"}) + + populate_stats(site, [ + build(:pageview, + referrer_source: "Bing", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Google", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Register", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "event:goal", ["Register"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [1]} + ] + end + + test "wildcard pageview goal filter for breakdown by event:page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/register"), + build(:pageview, pathname: "/en/register", user_id: @user_id), + build(:pageview, pathname: "/en/register", user_id: @user_id), + build(:pageview, pathname: "/123/it/register"), + build(:pageview, pathname: "/should-not-appear") + ]) + + insert(:goal, %{site: site, page_path: "/**register"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "pageviews"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["matches", "event:goal", ["Visit /**register"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/en/register"], "metrics" => [2, 3]}, + %{"dimensions" => ["/123/it/register"], "metrics" => [1, 1]} + ] + end + + test "mixed multi-goal filter for breakdown by visit:country", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, country_code: "EE", pathname: "/en/register"), + build(:event, country_code: "EE", name: "Signup", pathname: "/en/register"), + build(:pageview, country_code: "US", pathname: "/123/it/register"), + build(:pageview, country_code: "US", pathname: "/different") + ]) + + insert(:goal, %{site: site, page_path: "/**register"}) + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "pageviews", "events"], + "date_range" => "all", + "dimensions" => ["visit:country"], + "filters" => [ + ["matches", "event:goal", ["Signup", "Visit /**register"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["EE"], "metrics" => [2, 1, 2]}, + %{"dimensions" => ["US"], "metrics" => [1, 1, 1]} + ] + end + + test "event:goal custom event filter for breakdown by event page", %{conn: conn, site: site} do + insert(:goal, %{site: site, event_name: "Register"}) + + populate_stats(site, [ + build(:event, + pathname: "/en/register", + name: "Register" + ), + build(:event, + pathname: "/en/register", + name: "Register" + ), + build(:event, + pathname: "/it/register", + name: "Register" + ) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "property" => "event:page", + "filters" => "event:goal == Register" + }) + + assert json_response(conn, 200) == %{ + "results" => [ + %{"page" => "/en/register", "visitors" => 2}, + %{"page" => "/it/register", "visitors" => 1} + ] + } + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "event:goal", ["Register"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/en/register"], "metrics" => [2]}, + %{"dimensions" => ["/it/register"], "metrics" => [1]} + ] + end + + test "IN filter for event:page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/ignore", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/important-page", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "event:page", ["/plausible.io", "/important-page"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/plausible.io"], "metrics" => [2]}, + %{"dimensions" => ["/important-page"], "metrics" => [1]} + ] + end + + test "IN filter for visit:browser", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/ignore", + browser: "Firefox", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/important-page", + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "visit:browser", ["Chrome", "Safari"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/plausible.io"], "metrics" => [2]}, + %{"dimensions" => ["/important-page"], "metrics" => [1]} + ] + end + + test "IN filter for visit:entry_page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/ignore", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/important-page", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "event:page", ["/plausible.io", "/important-page"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/plausible.io"], "metrics" => [100]}, + %{"dimensions" => ["/important-page"], "metrics" => [100]} + ] + end + + test "IN filter for event:name", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Login", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Irrelevant", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"], + "filters" => [ + ["is", "event:name", ["Signup", "Login"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Signup"], "metrics" => [2]}, + %{"dimensions" => ["Login"], "metrics" => [1]} + ] + end + + test "IN filter for event:props:*", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Safari", + "meta.key": ["browser"], + "meta.value": ["Safari"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["browser"], + "meta.value": ["Firefox"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:props:browser", ["Chrome", "Safari"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Chrome"], "metrics" => [2]}, + %{"dimensions" => ["Safari"], "metrics" => [1]} + ] + end + + test "Multiple event:props:* filters", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + "meta.key": ["browser", "prop"], + "meta.value": ["Chrome", "xyz"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Safari", + "meta.key": ["browser", "prop"], + "meta.value": ["Safari", "target_value"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["browser", "prop"], + "meta.value": ["Firefox", "target_value"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:props:browser", ["Chrome", "Safari"]], + ["is", "event:props:prop", ["target_value"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Safari"], "metrics" => [1]} + ] + end + + test "IN filter for event:props:* including (none) value", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["browser"], + "meta.value": ["Firefox"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [["is", "event:props:browser", ["Chrome", "(none)"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Chrome"], "metrics" => [2]}, + %{"dimensions" => ["Safari"], "metrics" => [1]} + ] + end + + test "can use a is_not filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, browser: "Chrome"), + build(:pageview, browser: "Safari"), + build(:pageview, browser: "Safari"), + build(:pageview, browser: "Edge") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is_not", "visit:browser", ["Chrome"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Safari"], "metrics" => [2]}, + %{"dimensions" => ["Edge"], "metrics" => [1]} + ] + end + + describe "metrics" do + test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "Signup", user_id: 1), + build(:event, name: "Signup", user_id: 1), + build(:pageview, pathname: "/blog"), + build(:pageview, pathname: "/blog/post"), + build(:pageview) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events", "conversion_rate"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]}, + %{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]} + ] + end + + test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "Signup", user_id: 1), + build(:pageview) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["conversion_rate"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Signup"], "metrics" => [50.0]} + ] + end + + test "returns conversion_rate in a goal filtered custom prop breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]), + build(:pageview, + pathname: "/blog/2", + "meta.key": ["author"], + "meta.value": ["Marko"], + user_id: 1 + ), + build(:pageview, + pathname: "/blog/3", + "meta.key": ["author"], + "meta.value": ["Marko"], + user_id: 1 + ), + build(:pageview, pathname: "/blog"), + build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]), + build(:pageview) + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "filters" => [["matches", "event:goal", ["Visit /blog**"]]], + "metrics" => ["visitors", "events", "conversion_rate"], + "dimensions" => ["event:props:author"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]}, + %{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]}, + %{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]} + ] + end + + test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview) + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:author"], + "filters" => [["matches", "event:goal", ["Visit /blog**"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Uku"], "metrics" => [50]} + ] + end + + test "returns conversion_rate in a goal filtered event:page breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, pathname: "/en/register", name: "pageview"), + build(:event, pathname: "/en/register", name: "Signup"), + build(:event, pathname: "/en/register", name: "Signup"), + build(:event, pathname: "/it/register", name: "Signup", user_id: 1), + build(:event, pathname: "/it/register", name: "Signup", user_id: 1), + build(:event, pathname: "/it/register", name: "pageview") + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [["is", "event:goal", ["Signup"]]], + "metrics" => ["visitors", "events", "group_conversion_rate"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]}, + %{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]} + ] + end + + test "returns conversion_rate alone in a goal filtered event:page breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, pathname: "/en/register", name: "pageview"), + build(:event, pathname: "/en/register", name: "Signup") + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]], + "metrics" => ["group_conversion_rate"], + "dimensions" => ["event:page"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/en/register"], "metrics" => [50.0]} + ] + end + + test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, screen_size: "Mobile", name: "pageview"), + build(:event, screen_size: "Mobile", name: "AddToCart"), + build(:event, screen_size: "Mobile", name: "AddToCart"), + build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1), + build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1), + build(:event, screen_size: "Desktop", name: "pageview") + ]) + + # Make sure that revenue goals are treated the same + # way as regular custom event goals + insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR}) + insert(:goal, %{site: site, event_name: "AddToCart"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events", "group_conversion_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]}, + %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]} + ] + end + + test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, screen_size: "Mobile", name: "pageview"), + build(:event, screen_size: "Mobile", name: "AddToCart") + ]) + + insert(:goal, %{site: site, event_name: "AddToCart"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [["is", "event:goal", ["AddToCart"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Mobile"], "metrics" => [50]} + ] + end + + test "all metrics for breakdown by visit prop", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: 1, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "signup", + user_id: 1, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:05:00] + ), + build(:pageview, + user_id: 1, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:10:00] + ), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + referrer_source: "Twitter", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => [ + "visitors", + "visits", + "pageviews", + "events", + "bounce_rate", + "visit_duration" + ], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [2, 2, 3, 4, 50, 300]}, + %{"dimensions" => ["Twitter"], "metrics" => [1, 1, 1, 1, 100, 0]} + ] + end + + test "metrics=bounce_rate does not add visits to the response", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: 1, + pathname: "/entry-page-1", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: 1, + pathname: "/some-page", + timestamp: ~N[2021-01-01 00:10:00] + ), + build(:pageview, + user_id: 2, + pathname: "/entry-page-2", + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:05:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["visit:entry_page"], + "order_by" => [["visit:entry_page", "asc"]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/entry-page-1"], "metrics" => [0]}, + %{"dimensions" => ["/entry-page-2"], "metrics" => [100]} + ] + end + + test "filter by custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Safari", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["personal"], + browser: "IE", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:name", ["Purchase"]], + ["is", "event:props:package", ["business"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Safari"], "metrics" => [2]}, + %{"dimensions" => ["Chrome"], "metrics" => [1]} + ] + end + + test "all metrics for breakdown by event prop", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: 1, + pathname: "/", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: 1, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:10:00] + ), + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => [ + "visitors", + "visits", + "pageviews", + "events", + "bounce_rate", + "visit_duration" + ], + "date_range" => "all", + "dimensions" => ["event:page"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/"], "metrics" => [2, 2, 2, 2, 50, 300]}, + %{"dimensions" => ["/plausible.io"], "metrics" => [2, 2, 2, 2, 100, 0]} + ] + end + end + + describe "imported data" do + test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + screen_size: "Mobile" + ), + build(:imported_devices, + device: "Mobile", + visitors: 3, + pageviews: 5, + date: ~D[2021-01-01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "pageviews"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [ + ["is", "visit:device", ["Mobile"]] + ], + "include" => %{"imports" => true} + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}] + end + + test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + insert(:goal, site: site, event_name: "Purchase") + insert(:goal, site: site, page_path: "/test") + + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + pathname: "/test" + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:imported_custom_events, + name: "Purchase", + visitors: 3, + events: 5, + date: ~D[2021-01-01] + ), + build(:imported_pages, + page: "/test", + visitors: 2, + pageviews: 2, + date: ~D[2021-01-01] + ), + build(:imported_visitors, visitors: 5, date: ~D[2021-01-01]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "dimensions" => ["event:goal"], + "metrics" => ["visitors", "events", "pageviews", "conversion_rate"], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]}, + %{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]} + ] + end + + test "pageviews are returned as events for breakdown reports other than custom events", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_operating_systems, + operating_system: "Mac", + pageviews: 1, + date: ~D[2021-01-01] + ), + build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01]) + ]) + + breakdown_and_first = fn dimension -> + conn + |> post("/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["events"], + "date_range" => ["2021-01-01", "2021-01-01"], + "dimensions" => [dimension], + "include" => %{"imports" => true} + }) + |> json_response(200) + |> Map.get("results") + |> List.first() + end + + assert %{"dimensions" => ["Chrome"], "metrics" => [1]} = + breakdown_and_first.("visit:browser") + + assert %{"dimensions" => ["Desktop"], "metrics" => [1]} = + breakdown_and_first.("visit:device") + + assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country") + assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os") + assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page") + + assert %{"dimensions" => ["Google"], "metrics" => [1]} = + breakdown_and_first.("visit:source") + end + + for goal_name <- Plausible.Imported.goals_with_url() do + test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do + insert(:goal, event_name: unquote(goal_name), site: site) + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: unquote(goal_name), + "meta.key": ["url"], + "meta.value": ["https://one.com"] + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 2, + events: 5, + link_url: "https://one.com" + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 5, + events: 10, + link_url: "https://two.com" + ), + build(:imported_custom_events, + name: "some goal", + visitors: 5, + events: 10 + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events", "conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:url"], + "filters" => [ + ["is", "event:goal", [unquote(goal_name)]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]}, + %{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]} + ] + + refute json_response(conn, 200)["meta"]["warning"] + end + end + + for goal_name <- Plausible.Imported.goals_with_path() do + test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do + insert(:goal, event_name: unquote(goal_name), site: site) + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: unquote(goal_name), + "meta.key": ["path"], + "meta.value": ["/one"] + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 2, + events: 5, + path: "/one" + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 5, + events: 10, + path: "/two" + ), + build(:imported_custom_events, + name: "some goal", + visitors: 5, + events: 10 + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events", "conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:path"], + "filters" => [ + ["is", "event:goal", [unquote(goal_name)]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/two"], "metrics" => [5, 10, 50]}, + %{"dimensions" => ["/one"], "metrics" => [3, 6, 30]} + ] + + refute json_response(conn, 200)["meta"]["warning"] + end + end + + test "adds a warning when query params are not supported for imported data", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + insert(:goal, event_name: "Signup", site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: "Signup", + "meta.key": ["package"], + "meta.value": ["large"] + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["large"], "metrics" => [1]} + ] + + assert json_response(conn, 200)["meta"]["warning"] =~ + "Imported stats are not included in the results because query parameters are not supported." + end + + test "does not add a warning when there are no site imports", %{conn: conn, site: site} do + insert(:goal, event_name: "Signup", site: site) + + populate_stats(site, [ + build(:event, + name: "Signup", + "meta.key": ["package"], + "meta.value": ["large"] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ], + "include" => %{"imports" => true} + }) + + refute json_response(conn, 200)["meta"]["warning"] + end + + test "does not add a warning when import is out of queried date range", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3)) + + insert(:goal, event_name: "Signup", site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: "Signup", + "meta.key": ["package"], + "meta.value": ["large"] + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "day", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ], + "include" => %{"imports" => true} + }) + + refute json_response(conn, 200)["meta"]["warning"] + end + + test "applies multiple filters if the properties belong to the same table", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"), + build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"), + build(:imported_sources, + source: "Facebook", + utm_medium: "something_else", + utm_term: "one" + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "day", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "visit:utm_medium", ["organic"]], + ["is", "visit:utm_term", ["one"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Google"], "metrics" => [1]} + ] + end + + test "ignores imported data if filtered property belongs to a different table than the breakdown property", + %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_sources, source: "Google"), + build(:imported_devices, device: "Desktop") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "day", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "visit:device", ["Desktop"]] + ], + "include" => %{"imports" => true} + }) + + assert %{ + "results" => [], + "meta" => meta + } = json_response(conn, 200) + + assert meta["warning"] =~ "Imported stats are not included in the results" + end + + test "imported country, region and city data", + %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:15:00], + country_code: "DE", + subdivision1_code: "DE-BE", + city_geoname_id: 2_950_159 + ), + build(:pageview, + timestamp: ~N[2021-01-01 00:15:00], + country_code: "DE", + subdivision1_code: "DE-BE", + city_geoname_id: 2_950_159 + ), + build(:pageview, + timestamp: ~N[2021-01-01 00:15:00], + country_code: "EE", + subdivision1_code: "EE-37", + city_geoname_id: 588_409 + ), + build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33) + ]) + + for {dimension, stats_value, imports_value} <- [ + {"visit:country", "DE", "EE"}, + {"visit:region", "DE-BE", "EE-37"}, + {"visit:city", 2_950_159, 588_409} + ] do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => [dimension], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => [imports_value], "metrics" => [34]}, + %{"dimensions" => [stats_value], "metrics" => [2]} + ] + end + end + end + + test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-02 00:00:00] + ), + build(:pageview, timestamp: ~N[2021-01-03 00:00:00]), + build(:pageview, + referrer_source: "Twitter", + timestamp: ~N[2021-01-03 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2021-01-01", "2021-01-03"], + "dimensions" => ["time", "visit:source"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-01T00:00:00Z", "Google"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-02T00:00:00Z", "Google"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-02T00:00:00Z", "Direct / None"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-03T00:00:00Z", "Direct / None"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-03T00:00:00Z", "Twitter"], "metrics" => [1]} + ] + end +end diff --git a/test/plausible_web/controllers/api/paddle_controller_test.exs b/test/plausible_web/controllers/api/paddle_controller_test.exs index 2c1398d53b51..47e4d1fc4927 100644 --- a/test/plausible_web/controllers/api/paddle_controller_test.exs +++ b/test/plausible_web/controllers/api/paddle_controller_test.exs @@ -2,7 +2,10 @@ defmodule PlausibleWeb.Api.PaddleControllerTest do use PlausibleWeb.ConnCase, async: true use Plausible.Repo - @body %{ + import Mox + setup :verify_on_exit! + + @webhook_body %{ "alert_id" => "16173800", "alert_name" => "subscription_created", "cancel_url" => @@ -29,15 +32,114 @@ defmodule PlausibleWeb.Api.PaddleControllerTest do describe "webhook verification" do test "is verified when signature is correct", %{conn: conn} do insert(:user, id: 235) - conn = post(conn, "/api/paddle/webhook", @body) + conn = post(conn, Routes.paddle_path(conn, :webhook), @webhook_body) assert conn.status == 200 end test "not verified when signature is corrupted", %{conn: conn} do - corrupted = Map.put(@body, "p_signature", Base.encode64("123 fake signature")) - conn = post(conn, "/api/paddle/webhook", corrupted) + corrupted = Map.put(@webhook_body, "p_signature", Base.encode64("123 fake signature")) + conn = post(conn, Routes.paddle_path(conn, :webhook), corrupted) assert conn.status == 400 end end + + describe "fetching currency" do + test "retrieves successfully", %{conn: conn} do + expect_get_prices_response(get_prices_body("USD")) + + conn = get(conn, Routes.paddle_path(conn, :currency)) + assert_receive :paddle_queried + assert json_response(conn, 200) == %{"currency" => "$"} + end + + test "caches per ip", %{conn: initial_conn} do + expect_get_prices_response(get_prices_body("USD")) + + conn = get(initial_conn, Routes.paddle_path(initial_conn, :currency)) + assert json_response(conn, 200) == %{"currency" => "$"} + assert_receive :paddle_queried + + expect_get_prices_response(get_prices_body("GBP")) + + conn = get(initial_conn, Routes.paddle_path(initial_conn, :currency)) + assert json_response(conn, 200) == %{"currency" => "$"} + refute_receive :paddle_queried + + new_ip = + Plug.Conn.put_req_header(initial_conn, "x-forwarded-for", Plausible.TestUtils.random_ip()) + + conn = get(new_ip, Routes.paddle_path(initial_conn, :currency)) + assert json_response(conn, 200) == %{"currency" => "£"} + assert_receive :paddle_queried + end + + test "falls back to EUR when paddle fails to respond", %{conn: conn} do + expect_get_prices_response(%{"response" => %{}}) + + conn = get(conn, Routes.paddle_path(conn, :currency)) + assert_receive :paddle_queried + assert json_response(conn, 200) == %{"currency" => "€"} + end + + test "does not cache failed fetches", %{conn: initial_conn} do + expect_get_prices_response(%{"response" => %{}}) + + conn = get(initial_conn, Routes.paddle_path(initial_conn, :currency)) + assert json_response(conn, 200) == %{"currency" => "€"} + + expect_get_prices_response(get_prices_body("USD")) + + conn = get(initial_conn, Routes.paddle_path(initial_conn, :currency)) + assert json_response(conn, 200) == %{"currency" => "$"} + end + + defp expect_get_prices_response(body) do + test = self() + + expect( + Plausible.HTTPClient.Mock, + :get, + fn "https://checkout.paddle.com/api/2.0/prices", + _, + %{customer_ip: _customer_ip, product_ids: "857097"} -> + send(test, :paddle_queried) + + {:ok, + %Finch.Response{ + status: 200, + headers: [{"content-type", "application/json"}], + body: body + }} + end + ) + end + + defp get_prices_body(currency) do + %{ + "response" => %{ + "customer_country" => "PL", + "products" => [ + %{ + "applied_coupon" => [], + "currency" => currency, + "list_price" => %{"gross" => 49.0, "net" => 49.0, "tax" => 0.0}, + "price" => %{"gross" => 49.0, "net" => 49.0, "tax" => 0.0}, + "product_id" => 857_097, + "product_title" => "random", + "subscription" => %{ + "frequency" => 1, + "interval" => "month", + "list_price" => %{"gross" => 49.0, "net" => 49.0, "tax" => 0.0}, + "price" => %{"gross" => 49.0, "net" => 49.0, "tax" => 0.0}, + "trial_days" => 0 + }, + "vendor_set_prices_included_tax" => false + } + ] + }, + "success" => true + } + end + end end diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index 24e18ed2da32..a5bcdb38b223 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -207,6 +207,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do ] end + @tag capture_log: true test "garbage filters don't crash the call", %{conn: conn, site: site} do filters = "{\"source\":\"Direct / None\",\"screen\":\"Desktop\",\"browser\":\"Chrome\",\"os\":\"Mac\",\"os_version\":\"10.15\",\"country\":\"DE\",\"city\":\"2950159\"}%' AND 2*3*8=6*8 AND 'L9sv'!='L9sv%" diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs index 370eb0ecf6b6..16e7a35f926d 100644 --- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs @@ -602,15 +602,15 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do assert json_response(conn, 200)["results"] == [ %{ - "name" => "blog", + "name" => "ad", "visitors" => 2, - "bounce_rate" => 50.0, + "bounce_rate" => 100.0, "visit_duration" => 50.0 }, %{ - "name" => "ad", + "name" => "blog", "visitors" => 2, - "bounce_rate" => 100.0, + "bounce_rate" => 50.0, "visit_duration" => 50.0 } ] @@ -708,7 +708,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do assert json_response(conn, 200)["results"] == [ %{ - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => 60, "visitors" => 3, "pageviews" => 4, diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index 773bc8cbf534..d4b0c36e8d7b 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -1241,8 +1241,8 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do end end - @tag :ee_only describe "GET /api/stats/main-graph - total_revenue plot" do + @describetag :ee_only setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import] test "plots total_revenue for a month", %{conn: conn, site: site} do @@ -1321,8 +1321,8 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do end end - @tag :ee_only describe "GET /api/stats/main-graph - average_revenue plot" do + @describetag :ee_only setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import] test "plots total_revenue for a month", %{conn: conn, site: site} do diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 4af5c5cd0bd5..6a55d0bf841d 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -340,7 +340,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/blog/other-post", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -392,7 +392,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/blog/other-post", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -744,7 +744,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/blog/post-2", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -789,7 +789,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/blog/(/post-2", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -842,7 +842,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/about", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -940,7 +940,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/" }, %{ - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil, "visitors" => 1, "pageviews" => 1, @@ -1066,7 +1066,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "visitors" => 2 }, %{ - "bounce_rate" => nil, + "bounce_rate" => 0, "name" => "/exit-blog", "pageviews" => 1, "time_on_page" => nil, @@ -1192,7 +1192,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/" }, %{ - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => 60, "visitors" => 2, "pageviews" => 2, diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index a76b1da27415..ea7b9c15a2ea 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -453,6 +453,9 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do referrer_source: "DuckDuckGo", referrer: "duckduckgo.com" ), + build(:imported_sources, + source: "DuckDuckGo" + ), build(:imported_sources, source: "DuckDuckGo" ) @@ -467,7 +470,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true") assert json_response(conn, 200)["results"] == [ - %{"name" => "DuckDuckGo", "visitors" => 2} + %{"name" => "Google", "visitors" => 2} ] end @@ -590,17 +593,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ) assert json_response(conn, 200)["results"] == [ - %{ - "name" => "social", - "visitors" => 1, - "bounce_rate" => 0, - "visit_duration" => 900 - }, %{ "name" => "email", "visitors" => 1, "bounce_rate" => 100, "visit_duration" => 0 + }, + %{ + "name" => "social", + "visitors" => 1, + "bounce_rate" => 0, + "visit_duration" => 900 } ] @@ -612,16 +615,16 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do assert json_response(conn, 200)["results"] == [ %{ - "name" => "social", + "name" => "email", "visitors" => 2, "bounce_rate" => 50, - "visit_duration" => 800.0 + "visit_duration" => 50 }, %{ - "name" => "email", + "name" => "social", "visitors" => 2, "bounce_rate" => 50, - "visit_duration" => 50 + "visit_duration" => 800.0 } ] end diff --git a/test/plausible_web/controllers/auth_controller_test.exs b/test/plausible_web/controllers/auth_controller_test.exs index 32383f44fcd8..73b5138d3dd3 100644 --- a/test/plausible_web/controllers/auth_controller_test.exs +++ b/test/plausible_web/controllers/auth_controller_test.exs @@ -938,9 +938,6 @@ defmodule PlausibleWeb.AuthControllerTest do doc = get(conn, "/settings") |> html_response(200) - assert text_of_attr(find(doc, "#monthly_pageview_usage_container"), "x-data") == - "{ tab: 'current_cycle' }" - assert class_of_element(doc, "#billing_cycle_tab_penultimate_cycle button") =~ "pointer-events-none" @@ -948,72 +945,20 @@ defmodule PlausibleWeb.AuthControllerTest do end @tag :ee_only - test "penultimate and last cycles are both disabled if there's no usage", %{ + test "last cycle tab is selected by default", %{ conn: conn, user: user } do - site = insert(:site, members: [user]) - - populate_stats(site, [ - build(:event, name: "pageview", timestamp: Timex.shift(Timex.now(), days: -5)) - ]) - - last_bill_date = Timex.shift(Timex.today(), days: -10) - insert(:subscription, paddle_plan_id: @v4_plan_id, user: user, - last_bill_date: last_bill_date + last_bill_date: Timex.shift(Timex.today(), days: -1) ) doc = get(conn, "/settings") |> html_response(200) assert text_of_attr(find(doc, "#monthly_pageview_usage_container"), "x-data") == - "{ tab: 'current_cycle' }" - - assert class_of_element(doc, "#billing_cycle_tab_last_cycle button") =~ - "pointer-events-none" - - assert text_of_element(doc, "#billing_cycle_tab_last_cycle") =~ "Not available" - - assert class_of_element(doc, "#billing_cycle_tab_penultimate_cycle button") =~ - "pointer-events-none" - - assert text_of_element(doc, "#billing_cycle_tab_penultimate_cycle") =~ "Not available" - end - - @tag :ee_only - test "when last cycle usage is 0, it's still not disabled if penultimate cycle has usage", %{ - conn: conn, - user: user - } do - site = insert(:site, members: [user]) - - populate_stats(site, [ - build(:event, name: "pageview", timestamp: Timex.shift(Timex.now(), days: -5)), - build(:event, name: "pageview", timestamp: Timex.shift(Timex.now(), days: -50)) - ]) - - last_bill_date = Timex.shift(Timex.today(), days: -10) - - insert(:subscription, - paddle_plan_id: @v4_plan_id, - user: user, - last_bill_date: last_bill_date - ) - - doc = get(conn, "/settings") |> html_response(200) - - assert text_of_attr(find(doc, "#monthly_pageview_usage_container"), "x-data") == - "{ tab: 'current_cycle' }" - - refute class_of_element(doc, "#billing_cycle_tab_last_cycle") =~ "pointer-events-none" - refute text_of_element(doc, "#billing_cycle_tab_last_cycle") =~ "Not available" - - refute class_of_element(doc, "#billing_cycle_tab_penultimate_cycle") =~ - "pointer-events-none" - - refute text_of_element(doc, "#billing_cycle_tab_penultimate_cycle") =~ "Not available" + "{ tab: 'last_cycle' }" end @tag :ee_only diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index 27913c2accf2..96519be035d4 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -364,7 +364,7 @@ defmodule PlausibleWeb.SiteControllerTest do }) assert redirected_to(conn) == "/example.com/snippet?site_created=true" - assert Plausible.Billing.Quota.site_usage(user) == 3 + assert Plausible.Billing.Quota.Usage.site_usage(user) == 3 end for url <- ["https://Example.com/", "HTTPS://EXAMPLE.COM/", "/Example.com/", "//Example.com/"] do @@ -436,6 +436,31 @@ defmodule PlausibleWeb.SiteControllerTest do assert html_response(conn, 200) =~ "This domain cannot be registered. Perhaps one of your colleagues registered it?" end + + test "allows creating the site if domain was changed by the owner", %{ + conn: conn, + user: user + } do + :site + |> insert( + domain: "example.com", + memberships: [ + build(:site_membership, user: user, role: :owner) + ] + ) + |> Plausible.Site.Domain.change("new.example.com") + + conn = + post(conn, "/sites", %{ + "site" => %{ + "domain" => "example.com", + "timezone" => "Europe/London" + } + }) + + assert redirected_to(conn) == + "/example.com/snippet?site_created=true" + end end describe "GET /:website/snippet" do @@ -463,6 +488,31 @@ defmodule PlausibleWeb.SiteControllerTest do end end + describe "GET /:website/settings/people" do + setup [:create_user, :log_in, :create_site] + + @tag :ee_only + test "shows members page with links to CRM for super admin", %{ + conn: conn, + user: user, + site: site + } do + patch_env(:super_admin_user_ids, [user.id]) + + conn = get(conn, "/#{site.domain}/settings/people") + resp = html_response(conn, 200) + + assert resp =~ "/crm/auth/user/#{user.id}" + end + + test "does not show CRM links to non-super admin user", %{conn: conn, user: user, site: site} do + conn = get(conn, "/#{site.domain}/settings/people") + resp = html_response(conn, 200) + + refute resp =~ "/crm/auth/user/#{user.id}" + end + end + describe "GET /:website/settings/goals" do setup [:create_user, :log_in, :create_site] diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index 5d090e97b71e..a2a95ab76268 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -99,6 +99,11 @@ defmodule PlausibleWeb.StatsControllerTest do conn = get(conn, "/" <> site.domain) assert html_response(conn, 404) =~ "There's nothing here" end + + test "does not show CRM link to the site", %{conn: conn, site: site} do + conn = get(conn, "/" <> site.domain) + refute html_response(conn, 200) =~ "/crm/sites/site/#{site.id}" + end end describe "GET /:website - as a super admin" do @@ -152,6 +157,12 @@ defmodule PlausibleWeb.StatsControllerTest do [{"div", attrs, _}] = find(resp, @react_container) assert Enum.all?(attrs, fn {k, v} -> is_binary(k) and is_binary(v) end) end + + test "shows CRM link to the site", %{conn: conn} do + site = insert(:site) + conn = get(conn, "/" <> site.domain) + assert html_response(conn, 200) =~ "/crm/sites/site/#{site.id}" + end end defp make_user_super_admin(%{user: user}) do diff --git a/test/plausible_web/live/choose_plan_test.exs b/test/plausible_web/live/choose_plan_test.exs index 274e14655a0e..3104c7546305 100644 --- a/test/plausible_web/live/choose_plan_test.exs +++ b/test/plausible_web/live/choose_plan_test.exs @@ -8,6 +8,9 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do alias Plausible.{Repo, Billing.Subscription} @v1_10k_yearly_plan_id "572810" + @v1_50m_yearly_plan_id "650653" + @v2_20m_yearly_plan_id "653258" + @v4_growth_10k_yearly_plan_id "857079" @v4_growth_200k_yearly_plan_id "857081" @v4_business_5m_monthly_plan_id "857111" @v3_business_10k_monthly_plan_id "857481" @@ -358,16 +361,87 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do "https://plausible.io/white-label-web-analytics" end - test "displays usage", %{conn: conn, site: site} do + test "displays usage in the last cycle", %{conn: conn, site: site} do + yesterday = NaiveDateTime.utc_now() |> NaiveDateTime.add(-1, :day) + populate_stats(site, [ - build(:pageview), - build(:pageview) + build(:pageview, timestamp: yesterday), + build(:pageview, timestamp: yesterday) ]) {:ok, _lv, doc} = get_liveview(conn) assert doc =~ "You have used" assert doc =~ "2" - assert doc =~ "billable pageviews in the last 30 days" + assert doc =~ "billable pageviews in the last billing cycle" + end + + test "renders notice about pending ownerships and counts their usage", %{ + conn: conn, + user: user, + site: site + } do + yesterday = NaiveDateTime.utc_now() |> NaiveDateTime.add(-1, :day) + + populate_stats(site, [ + build(:pageview, timestamp: yesterday) + ]) + + another_user = insert(:user) + + pending_site = + insert(:site, + memberships: [ + build(:site_membership, role: :owner, user: another_user), + build(:site_membership, role: :admin, user: build(:user)), + build(:site_membership, role: :viewer, user: build(:user)), + build(:site_membership, role: :viewer, user: build(:user)) + ] + ) + + populate_stats(pending_site, [ + build(:pageview, timestamp: yesterday) + ]) + + insert(:invitation, + site: pending_site, + inviter: another_user, + email: user.email, + role: :owner + ) + + {:ok, _lv, doc} = get_liveview(conn) + + assert doc =~ "Your account has been invited to become the owner of a site" + + assert text_of_element(doc, @growth_plan_tooltip) == + "Your usage exceeds the following limit(s): Team member limit" + + assert doc =~ "2" + assert doc =~ "billable pageviews in the last billing cycle" + end + + test "warns about losing access to a feature used by a pending ownership site", %{ + conn: conn, + user: user + } do + another_user = insert(:user) + pending_site = insert(:site, members: [another_user]) + + Plausible.Props.allow(pending_site, ["author"]) + + insert(:invitation, + site: pending_site, + inviter: another_user, + email: user.email, + role: :owner + ) + + {:ok, _lv, doc} = get_liveview(conn) + + assert doc =~ "Your account has been invited to become the owner of a site" + + assert text_of_attr(find(doc, @growth_checkout_button), "onclick") =~ + "if (confirm(\"This plan does not support Custom Properties, which you are currently using. Please note that by subscribing to this plan you will lose access to this feature.\")) {window.location = " end test "gets default selected interval from current subscription plan", %{conn: conn} do @@ -375,9 +449,9 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do assert class_of_element(doc, @yearly_interval_button) =~ @interval_button_active_class end - test "gets default pageview limit from current subscription plan", %{conn: conn} do + test "sets pageview slider according to last cycle usage", %{conn: conn} do {:ok, _lv, doc} = get_liveview(conn) - assert text_of_element(doc, @slider_value) == "200k" + assert text_of_element(doc, @slider_value) == "10k" end test "pageview slider changes selected volume", %{conn: conn} do @@ -401,7 +475,9 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do end test "checkout button text and click-disabling CSS classes are dynamic", %{conn: conn} do - {:ok, lv, doc} = get_liveview(conn) + {:ok, lv, _doc} = get_liveview(conn) + + doc = set_slider(lv, "200k") assert text_of_element(doc, @growth_checkout_button) == "Currently on this plan" assert class_of_element(doc, @growth_checkout_button) =~ "pointer-events-none bg-gray-400" @@ -431,7 +507,7 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do growth_checkout_button = find(doc, @growth_checkout_button) assert text_of_attr(growth_checkout_button, "onclick") =~ - "if (true) {window.location = '#{Routes.billing_path(conn, :change_plan_preview, @v4_growth_200k_yearly_plan_id)}'}" + "if (true) {window.location = '#{Routes.billing_path(conn, :change_plan_preview, @v4_growth_10k_yearly_plan_id)}'}" set_slider(lv, "5M") doc = element(lv, @monthly_interval_button) |> render_click() @@ -446,9 +522,9 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do describe "for a user with a v4 business subscription plan" do setup [:create_user, :create_site, :log_in, :subscribe_v4_business] - test "gets default pageview limit from current subscription plan", %{conn: conn} do + test "sets pageview slider according to last cycle usage", %{conn: conn} do {:ok, _lv, doc} = get_liveview(conn) - assert text_of_element(doc, @slider_value) == "5M" + assert text_of_element(doc, @slider_value) == "10k" end test "makes it clear that the user is currently on a business tier", %{conn: conn} do @@ -462,11 +538,12 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do end test "checkout button text and click-disabling CSS classes are dynamic", %{conn: conn} do - {:ok, lv, doc} = get_liveview(conn) + {:ok, lv, _doc} = get_liveview(conn) + + doc = set_slider(lv, "5M") assert text_of_element(doc, @business_checkout_button) == "Currently on this plan" assert class_of_element(doc, @business_checkout_button) =~ "pointer-events-none bg-gray-400" - assert text_of_element(doc, @growth_checkout_button) == "Downgrade to Growth" doc = element(lv, @yearly_interval_button) |> render_click() @@ -634,7 +711,10 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do test "checkout buttons are disabled + notice about billing details (unless plan owned already)", %{conn: conn} do - {:ok, lv, doc} = get_liveview(conn) + {:ok, lv, _doc} = get_liveview(conn) + + doc = set_slider(lv, "200k") + assert class_of_element(doc, @growth_checkout_button) =~ "pointer-events-none bg-gray-400" assert text_of_element(doc, @growth_checkout_button) =~ "Currently on this plan" refute element_exists?(doc, "#{@growth_checkout_button} + p") @@ -664,7 +744,10 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do test "checkout buttons are disabled + notice about billing details when plan not owned already", %{conn: conn} do - {:ok, lv, doc} = get_liveview(conn) + {:ok, lv, _doc} = get_liveview(conn) + + doc = set_slider(lv, "200k") + assert class_of_element(doc, @growth_checkout_button) =~ "pointer-events-none bg-gray-400" assert text_of_element(doc, @growth_checkout_button) =~ "Currently on this plan" refute element_exists?(doc, "#{@growth_checkout_button} + p") @@ -716,16 +799,13 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do end end - describe "for a grandfathered user" do + describe "for a grandfathered user with a high volume plan" do setup [:create_user, :create_site, :log_in] - setup %{user: user} = context do - create_subscription_for(user, paddle_plan_id: @v1_10k_yearly_plan_id) - {:ok, context} - end + test "on a 50M v1 plan, Growth tiers are available at 20M, 50M, 50M+, but Business tiers are not", + %{conn: conn, user: user} do + create_subscription_for(user, paddle_plan_id: @v1_50m_yearly_plan_id) - test "on a v1 plan, Growth tiers are available at 20M, 50M, 50M+, but Business tiers are not", - %{conn: conn} do {:ok, lv, _doc} = get_liveview(conn) doc = set_slider(lv, 8) @@ -751,6 +831,42 @@ defmodule PlausibleWeb.Live.ChoosePlanTest do refute text_of_element(doc, @growth_plan_box) =~ "Contact us" end + test "on a 20M v2 plan, Growth tiers are available at 20M and 20M+, but not 50M", + %{conn: conn, user: user} do + create_subscription_for(user, paddle_plan_id: @v2_20m_yearly_plan_id) + + {:ok, lv, _doc} = get_liveview(conn) + + doc = set_slider(lv, 8) + assert text_of_element(doc, @slider_value) == "20M" + assert text_of_element(doc, @business_plan_box) =~ "Contact us" + assert text_of_element(doc, @growth_price_tag_amount) == "€900" + assert text_of_element(doc, @growth_price_tag_interval) == "/year" + + doc = set_slider(lv, 9) + assert text_of_element(doc, @slider_value) == "20M+" + assert text_of_element(doc, @business_plan_box) =~ "Contact us" + assert text_of_element(doc, @growth_plan_box) =~ "Contact us" + end + end + + describe "for a grandfathered user on a v1 10k plan" do + setup [:create_user, :create_site, :log_in] + + setup %{user: user} = context do + create_subscription_for(user, paddle_plan_id: @v1_10k_yearly_plan_id) + {:ok, context} + end + + test "v1 20M and 50M Growth plans are not available", + %{conn: conn} do + {:ok, lv, _doc} = get_liveview(conn) + + doc = set_slider(lv, 8) + assert text_of_element(doc, @slider_value) == "10M+" + assert text_of_element(doc, @growth_plan_box) =~ "Contact us" + end + test "displays grandfathering notice in the Growth box instead of benefits", %{conn: conn} do {:ok, _lv, doc} = get_liveview(conn) growth_box = text_of_element(doc, @growth_plan_box) diff --git a/test/plausible_web/live/components/combo_box_test.exs b/test/plausible_web/live/components/combo_box_test.exs index 156c92025780..eca08effa82f 100644 --- a/test/plausible_web/live/components/combo_box_test.exs +++ b/test/plausible_web/live/components/combo_box_test.exs @@ -1,6 +1,7 @@ defmodule PlausibleWeb.Live.Components.ComboBoxTest do use PlausibleWeb.ConnCase, async: true - import Phoenix.LiveViewTest + import Phoenix.LiveViewTest, except: [render_component: 2] + import Plausible.LiveViewTest, only: [render_component: 2] import Plausible.Test.Support.HTML alias PlausibleWeb.Live.Components.ComboBox diff --git a/test/plausible_web/live/components/form_test.exs b/test/plausible_web/live/components/form_test.exs index 04d060549e6f..c344add90462 100644 --- a/test/plausible_web/live/components/form_test.exs +++ b/test/plausible_web/live/components/form_test.exs @@ -1,6 +1,6 @@ defmodule PlausibleWeb.Live.Components.FormTest do use PlausibleWeb.ConnCase, async: true - import Phoenix.LiveViewTest + import Plausible.LiveViewTest, only: [render_component: 2] import Plausible.Test.Support.HTML alias Plausible.Auth.User diff --git a/test/plausible_web/live/components/verification_test.exs b/test/plausible_web/live/components/verification_test.exs index 44712121fe71..4f10a5ed039a 100644 --- a/test/plausible_web/live/components/verification_test.exs +++ b/test/plausible_web/live/components/verification_test.exs @@ -1,6 +1,6 @@ defmodule PlausibleWeb.Live.Components.VerificationTest do use PlausibleWeb.ConnCase, async: true - import Phoenix.LiveViewTest + import Plausible.LiveViewTest, only: [render_component: 2] import Plausible.Test.Support.HTML @component PlausibleWeb.Live.Components.Verification diff --git a/test/plausible_web/live/goal_settings/form_test.exs b/test/plausible_web/live/goal_settings/form_test.exs index 8bf295908888..cca96d59af3c 100644 --- a/test/plausible_web/live/goal_settings/form_test.exs +++ b/test/plausible_web/live/goal_settings/form_test.exs @@ -31,18 +31,20 @@ defmodule PlausibleWeb.Live.GoalSettings.FormTest do lv = get_liveview(conn, site) html = render(lv) - [event_name, currency_display, currency_submit] = find(html, "#goals-form input") + [event_name_display, event_name, currency_display, currency_submit] = + find(html, "#goals-form-modalseq0 input") + assert name_of(event_name_display) == "display-event_name_input_modalseq0-tabseq0" assert name_of(event_name) == "goal[event_name]" - assert name_of(currency_display) == "display-currency_input" + assert name_of(currency_display) == "display-currency_input_modalseq0-tabseq0" assert name_of(currency_submit) == "goal[currency]" lv |> element(~s/a#pageview-tab/) |> render_click() html = render(lv) - [page_path_display, page_path] = find(html, "#goals-form input") - assert name_of(page_path_display) == "display-page_path_input" + [page_path_display, page_path] = find(html, "#goals-form-modalseq0 input") + assert name_of(page_path_display) == "display-page_path_input_modalseq0-tabseq1" assert name_of(page_path) == "goal[page_path]" end @@ -51,22 +53,22 @@ defmodule PlausibleWeb.Live.GoalSettings.FormTest do lv = get_liveview(conn, site) html = render(lv) - [event_name] = find(html, "#goals-form input") - + [event_name_display, event_name | _] = find(html, "#goals-form-modalseq0 input") + assert name_of(event_name_display) == "display-event_name_input_modalseq0-tabseq0" assert name_of(event_name) == "goal[event_name]" lv |> element(~s/a#pageview-tab/) |> render_click() html = render(lv) - [page_path_display, page_path] = find(html, "#goals-form input") - assert name_of(page_path_display) == "display-page_path_input" + [page_path_display, page_path] = find(html, "#goals-form-modalseq0 input") + assert name_of(page_path_display) == "display-page_path_input_modalseq0-tabseq1" assert name_of(page_path) == "goal[page_path]" end test "renders error on empty submission", %{conn: conn, site: site} do lv = get_liveview(conn, site) - lv |> element("#goals-form form") |> render_submit() + lv |> element("#goals-form-modalseq0 form") |> render_submit() html = render(lv) assert html =~ "this field is required and cannot be blank" @@ -79,7 +81,7 @@ defmodule PlausibleWeb.Live.GoalSettings.FormTest do refute render(lv) =~ "SampleCustomEvent" lv - |> element("#goals-form form") + |> element("#goals-form-modalseq0 form") |> render_submit(%{goal: %{event_name: "SampleCustomEvent"}}) html = render(lv) @@ -93,7 +95,7 @@ defmodule PlausibleWeb.Live.GoalSettings.FormTest do refute render(lv) =~ "SampleRevenueGoal" lv - |> element("#goals-form form") + |> element("#goals-form-modalseq0 form") |> render_submit(%{goal: %{event_name: "SampleRevenueGoal", currency: "EUR"}}) html = render(lv) @@ -105,7 +107,11 @@ defmodule PlausibleWeb.Live.GoalSettings.FormTest do test "creates a pageview goal", %{conn: conn, site: site} do lv = get_liveview(conn, site) refute render(lv) =~ "Visit /page/**" - lv |> element("#goals-form form") |> render_submit(%{goal: %{page_path: "/page/**"}}) + + lv + |> element("#goals-form-modalseq0 form") + |> render_submit(%{goal: %{page_path: "/page/**"}}) + html = render(lv) assert html =~ "Visit /page/**" assert html =~ "Pageview" @@ -119,13 +125,13 @@ defmodule PlausibleWeb.Live.GoalSettings.FormTest do test "currency combo works", %{conn: conn, site: site} do lv = get_liveview(conn, site) - type_into_combo(lv, "currency_input", "Polish") + type_into_combo(lv, "currency_input_modalseq0-tabseq0", "Polish") html = render(lv) assert element_exists?(html, ~s/a[phx-value-display-value="PLN - Polish Zloty"]/) refute element_exists?(html, ~s/a[phx-value-display-value="EUR - Euro"]/) - type_into_combo(lv, "currency_input", "Euro") + type_into_combo(lv, "currency_input_modalseq0-tabseq0", "Euro") html = render(lv) refute element_exists?(html, ~s/a[phx-value-display-value="PLN - Polish Zloty"]/) @@ -136,7 +142,7 @@ defmodule PlausibleWeb.Live.GoalSettings.FormTest do lv = get_liveview(conn, site) lv |> element(~s/a#pageview-tab/) |> render_click() - html = type_into_combo(lv, "page_path_input", "/hello") + html = type_into_combo(lv, "page_path_input_modalseq0-tabseq1", "/hello") assert html =~ "Create "/hello"" end @@ -150,14 +156,14 @@ defmodule PlausibleWeb.Live.GoalSettings.FormTest do lv = get_liveview(conn, site) lv |> element(~s/a#pageview-tab/) |> render_click() - type_into_combo(lv, "page_path_input", "/go/to/p") + type_into_combo(lv, "page_path_input_modalseq0-tabseq1", "/go/to/p") html = render(lv) assert html =~ "Create "/go/to/p"" assert html =~ "/go/to/page/1" refute html =~ "/go/home" - type_into_combo(lv, "page_path_input", "/go/h") + type_into_combo(lv, "page_path_input_modalseq0-tabseq1", "/go/h") html = render(lv) assert html =~ "/go/home" refute html =~ "/go/to/page/1" @@ -174,18 +180,57 @@ defmodule PlausibleWeb.Live.GoalSettings.FormTest do lv = get_liveview(conn, site) lv |> element(~s/a#pageview-tab/) |> render_click() - type_into_combo(lv, "page_path_input", "/go/to/p") + type_into_combo(lv, "page_path_input_modalseq0-tabseq1", "/go/to/p") html = render(lv) assert html =~ "Create "/go/to/p"" assert html =~ "/go/to/page/1" refute html =~ "/go/home" - type_into_combo(lv, "page_path_input", "/go/h") + type_into_combo(lv, "page_path_input_modalseq0-tabseq1", "/go/h") html = render(lv) assert html =~ "/go/home" refute html =~ "/go/to/page/1" end + + test "event name combo suggestions update on input", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "EventOne"), + build(:event, name: "EventTwo"), + build(:event, name: "EventThree") + ]) + + lv = get_liveview(conn, site) + + type_into_combo(lv, "event_name_input_modalseq0-tabseq0", "One") + html = render(lv) + + assert text_of_element(html, "#goals-form-modalseq0") =~ "EventOne" + refute text_of_element(html, "#goals-form-modalseq0") =~ "EventTwo" + refute text_of_element(html, "#goals-form-modalseq0") =~ "EventThree" + end + + test "event name combo suggestions are up to date after deletion", %{conn: conn, site: site} do + insert(:goal, site: site, event_name: "EventOne") + insert(:goal, site: site, event_name: "EventTwo") + insert(:goal, site: site, event_name: "EventThree") + + populate_stats(site, [ + build(:event, name: "EventOne"), + build(:event, name: "EventTwo"), + build(:event, name: "EventThree") + ]) + + lv = get_liveview(conn, site) + + # Delete the goal + goal = Plausible.Repo.get_by(Plausible.Goal, site_id: site.id, event_name: "EventOne") + html = lv |> element(~s/button#delete-goal-#{goal.id}/) |> render_click() + + assert text_of_element(html, "#goals-form-modalseq0") =~ "EventOne" + refute text_of_element(html, "#goals-form-modalseq0") =~ "EventTwo" + refute text_of_element(html, "#goals-form-modalseq0") =~ "EventThree" + end end defp type_into_combo(lv, id, text) do diff --git a/test/plausible_web/live/goal_settings_test.exs b/test/plausible_web/live/goal_settings_test.exs index c580adabd11a..985f3f9623b1 100644 --- a/test/plausible_web/live/goal_settings_test.exs +++ b/test/plausible_web/live/goal_settings_test.exs @@ -164,7 +164,7 @@ defmodule PlausibleWeb.Live.GoalSettingsTest do assert element_exists?( html, - ~s/div#goals-form form[phx-submit="save-goal"]/ + ~s/#goals-form-modal form[phx-submit="save-goal"]/ ) end @@ -189,7 +189,7 @@ defmodule PlausibleWeb.Live.GoalSettingsTest do # Add one goal lv - |> element("#goals-form form") + |> element("#goals-form-modal form") |> render_submit(%{goal: %{event_name: "Signup"}}) html = render(lv) diff --git a/test/plausible_web/plugins/api/controllers/capabilities_test.exs b/test/plausible_web/plugins/api/controllers/capabilities_test.exs index 162c7a5e75ec..250ff809fb24 100644 --- a/test/plausible_web/plugins/api/controllers/capabilities_test.exs +++ b/test/plausible_web/plugins/api/controllers/capabilities_test.exs @@ -85,6 +85,7 @@ defmodule PlausibleWeb.Plugins.API.Controllers.CapabilitiesTest do assert_schema(resp, "Capabilities", spec()) end + @tag :ee_only test "growth", %{conn: conn, site: site, token: token} do site = Plausible.Repo.preload(site, :owner) insert(:growth_subscription, user: site.owner) diff --git a/test/support/live_view_test_fix.ex b/test/support/live_view_test_fix.ex new file mode 100644 index 000000000000..2fba6e185814 --- /dev/null +++ b/test/support/live_view_test_fix.ex @@ -0,0 +1,42 @@ +defmodule Plausible.LiveViewTest do + @moduledoc """ + Temporary fix for `Phoenix.LiveViewTest.render_component/2` failing CI with warnings. + + This module can be removed once Plausible switches to `phoenix_live_view ~> 1.0.0` + """ + + @doc """ + Same as `Phoenix.LiveViewTest.render_component/2` but with backported fixes from + https://github.com/phoenixframework/phoenix_live_view/commit/489e8de024e03976e9ae38138eec517fbd456d27 + """ + defmacro render_component(component, assigns \\ Macro.escape(%{}), opts \\ []) do + endpoint = Module.get_attribute(__CALLER__.module, :endpoint) + + component = + if is_atom(component) do + quote do + unquote(component).__live__() + unquote(component) + end + else + component + end + + quote do + Plausible.LiveViewTest.__render_component__( + unquote(endpoint), + unquote(component), + unquote(assigns), + unquote(opts) + ) + end + end + + def __render_component__(endpoint, component, assigns, opts) when is_atom(component) do + Phoenix.LiveViewTest.__render_component__(endpoint, %{module: component}, assigns, opts) + end + + def __render_component__(endpoint, component, assigns, opts) do + Phoenix.LiveViewTest.__render_component__(endpoint, component, assigns, opts) + end +end diff --git a/test/support/paddle_api_mock.ex b/test/support/paddle_api_mock.ex index a2aee7f07820..b0aa46333264 100644 --- a/test/support/paddle_api_mock.ex +++ b/test/support/paddle_api_mock.ex @@ -75,7 +75,7 @@ defmodule Plausible.PaddleApi.Mock do # to give a reasonable testing structure for monthly and yearly plan # prices, this function returns prices with the following logic: # 10, 100, 20, 200, 30, 300, ...and so on. - def fetch_prices([_ | _] = product_ids) do + def fetch_prices([_ | _] = product_ids, _customer_ip) do {prices, _index} = Enum.reduce(product_ids, {%{}, 1}, fn p, {acc, i} -> price = diff --git a/test/workers/check_usage_test.exs b/test/workers/check_usage_test.exs index ba8246ef1a3f..e05be4f8d989 100644 --- a/test/workers/check_usage_test.exs +++ b/test/workers/check_usage_test.exs @@ -31,8 +31,8 @@ defmodule Plausible.Workers.CheckUsageTest do test "does not send an email if account has been over the limit for one billing month", %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 9_000}, @@ -46,7 +46,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert_no_emails_delivered() assert Repo.reload(user).grace_period == nil @@ -55,8 +55,8 @@ defmodule Plausible.Workers.CheckUsageTest do test "does not send an email if account is over the limit by less than 10%", %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 10_999}, @@ -70,7 +70,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert_no_emails_delivered() assert Repo.reload(user).grace_period == nil @@ -79,8 +79,8 @@ defmodule Plausible.Workers.CheckUsageTest do test "sends an email when an account is over their limit for two consecutive billing months", %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 11_000}, @@ -94,7 +94,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert_email_delivered_with( to: [user], @@ -107,8 +107,8 @@ defmodule Plausible.Workers.CheckUsageTest do test "sends an email suggesting enterprise plan when usage is greater than 10M ", %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 11_000_000}, @@ -122,7 +122,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert_delivered_email_matches(%{html_body: html_body}) @@ -136,8 +136,8 @@ defmodule Plausible.Workers.CheckUsageTest do |> Plausible.Auth.GracePeriod.start_changeset() |> Repo.update!() - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 11_000}, @@ -151,7 +151,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert_no_emails_delivered() assert Repo.reload(user).grace_period.id == existing_grace_period.id @@ -160,8 +160,8 @@ defmodule Plausible.Workers.CheckUsageTest do test "recommends a plan to upgrade to", %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 11_000}, @@ -175,7 +175,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert_delivered_email_matches(%{ html_body: html_body @@ -184,13 +184,74 @@ defmodule Plausible.Workers.CheckUsageTest do assert html_body =~ "We recommend you upgrade to the 100k/mo plan" end + test "clears grace period when plan is applicable again", %{user: user} do + usage_stub = + Plausible.Billing.Quota.Usage + |> stub(:monthly_pageview_usage, fn _user -> + %{ + penultimate_cycle: %{date_range: @date_range, total: 11_000}, + last_cycle: %{date_range: @date_range, total: 11_000} + } + end) + + insert(:subscription, + user: user, + paddle_plan_id: @paddle_id_10k, + last_bill_date: Timex.shift(Timex.today(), days: -1) + ) + + CheckUsage.perform(nil, usage_stub) + assert user |> Repo.reload() |> Plausible.Auth.GracePeriod.active?() + + usage_stub = + Plausible.Billing.Quota.Usage + |> stub(:monthly_pageview_usage, fn _user -> + %{ + penultimate_cycle: %{date_range: @date_range, total: 11_000}, + last_cycle: %{date_range: @date_range, total: 9_000} + } + end) + + CheckUsage.perform(nil, usage_stub) + refute user |> Repo.reload() |> Plausible.Auth.GracePeriod.active?() + end + describe "enterprise customers" do + test "skips checking enterprise users who already have a grace period", %{user: user} do + %{grace_period: existing_grace_period} = + user + |> Plausible.Auth.GracePeriod.start_manual_lock_changeset() + |> Repo.update!() + + usage_stub = + Plausible.Billing.Quota.Usage + |> stub(:monthly_pageview_usage, fn _user -> + %{ + penultimate_cycle: %{date_range: @date_range, total: 1_100_000}, + last_cycle: %{date_range: @date_range, total: 1_100_000} + } + end) + + enterprise_plan = insert(:enterprise_plan, user: user, monthly_pageview_limit: 1_000_000) + + insert(:subscription, + user: user, + paddle_plan_id: enterprise_plan.paddle_plan_id, + last_bill_date: Timex.shift(Timex.today(), days: -1) + ) + + CheckUsage.perform(nil, usage_stub) + + assert_no_emails_delivered() + assert Repo.reload(user).grace_period.id == existing_grace_period.id + end + test "checks billable pageview usage for enterprise customer, sends usage information to enterprise@plausible.io", %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 1_100_000}, @@ -206,7 +267,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert_email_delivered_with( to: [{nil, "enterprise@plausible.io"}], @@ -218,8 +279,8 @@ defmodule Plausible.Workers.CheckUsageTest do %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 1}, @@ -239,7 +300,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert_email_delivered_with( to: [{nil, "enterprise@plausible.io"}], @@ -248,8 +309,8 @@ defmodule Plausible.Workers.CheckUsageTest do end test "starts grace period when plan is outgrown", %{user: user} do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 1_100_000}, @@ -265,7 +326,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert user |> Repo.reload() |> Plausible.Auth.GracePeriod.active?() end end @@ -274,8 +335,8 @@ defmodule Plausible.Workers.CheckUsageTest do test "checks usage one day after the last_bill_date", %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 11_000}, @@ -289,7 +350,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: Timex.shift(Timex.today(), days: -1) ) - CheckUsage.perform(nil, quota_stub) + CheckUsage.perform(nil, usage_stub) assert_email_delivered_with( to: [user], @@ -300,8 +361,8 @@ defmodule Plausible.Workers.CheckUsageTest do test "does not check exactly one month after last_bill_date", %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 11_000}, @@ -315,7 +376,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: ~D[2021-03-28] ) - CheckUsage.perform(nil, quota_stub, ~D[2021-03-28]) + CheckUsage.perform(nil, usage_stub, ~D[2021-03-28]) assert_no_emails_delivered() end @@ -324,8 +385,8 @@ defmodule Plausible.Workers.CheckUsageTest do %{ user: user } do - quota_stub = - Plausible.Billing.Quota + usage_stub = + Plausible.Billing.Quota.Usage |> stub(:monthly_pageview_usage, fn _user -> %{ penultimate_cycle: %{date_range: @date_range, total: 11_000}, @@ -339,7 +400,7 @@ defmodule Plausible.Workers.CheckUsageTest do last_bill_date: ~D[2021-06-29] ) - CheckUsage.perform(nil, quota_stub, ~D[2021-08-30]) + CheckUsage.perform(nil, usage_stub, ~D[2021-08-30]) assert_email_delivered_with( to: [user],