diff --git a/backend/api.py b/backend/api.py index 5459948..9fcac84 100644 --- a/backend/api.py +++ b/backend/api.py @@ -20,6 +20,7 @@ ProvidersEnum, ) from backend.fetcher import ( + ContainerProvider, CoprProvider, KojiProvider, PackitProvider, @@ -58,7 +59,7 @@ @app.exception_handler(RequestValidationError) def _custom_http_exception_handler( request: Request, exc: HTTPException | RequestValidationError | Exception -): +) -> JSONResponse: if isinstance(exc, HTTPException): status_code = exc.status_code elif isinstance(exc, RequestValidationError): @@ -103,7 +104,9 @@ def review(request: Request): @app.get("/frontend/contribute/copr/{build_id}/{chroot}") @app.get("/frontend/contribute/koji/{build_id}/{chroot}") -def get_build_logs_with_chroot(request: Request, build_id: int, chroot: str): +def get_build_logs_with_chroot( + request: Request, build_id: int, chroot: str +) -> ContributeResponseSchema: provider_name = request.url.path.lstrip("/").split("/")[2] prov_kls = CoprProvider if provider_name == ProvidersEnum.copr else KojiProvider provider = prov_kls(build_id, chroot) @@ -148,6 +151,18 @@ def get_build_logs_from_url(base64: str) -> ContributeResponseSchema: ) +@app.get("/frontend/contribute/container/{base64}") +def get_logs_from_container(base64: str) -> ContributeResponseSchema: + build_url = b64decode(base64).decode("utf-8") + provider = ContainerProvider(build_url) + return ContributeResponseSchema( + build_id=None, + build_id_title=BuildIdTitleEnum.container, + build_url=build_url, + logs=provider.fetch_logs(), + ) + + # TODO: no response checking here, it will be deleted anyway @app.get("/frontend/contribute/debug") def get_debug_build_logs(): @@ -176,7 +191,12 @@ def _store_data_for_providers( feedback_input: FeedbackInputSchema, provider: ProvidersEnum, id_: int | str, *args ) -> OkResponse: storator = Storator3000(provider, id_) - result_to_store = schema_inp_to_out(feedback_input) + + if provider == ProvidersEnum.container: + result_to_store = schema_inp_to_out(feedback_input, is_with_spec=False) + else: + result_to_store = schema_inp_to_out(feedback_input) + storator.store(result_to_store) if len(args) > 0: rest = f"/{args[0]}" @@ -215,6 +235,13 @@ def contribute_review_url(feedback_input: FeedbackInputSchema, url: str) -> OkRe return _store_data_for_providers(feedback_input, ProvidersEnum.url, url) +@app.post("/frontend/contribute/container/{url}") +def contribute_review_container_logs( + feedback_input: FeedbackInputSchema, url: str +) -> OkResponse: + return _store_data_for_providers(feedback_input, ProvidersEnum.container, url) + + @app.get("/frontend/review") def frontend_review() -> FeedbackSchema: if os.environ.get("ENV") == "production": diff --git a/backend/constants.py b/backend/constants.py index 21ed071..5cf3433 100644 --- a/backend/constants.py +++ b/backend/constants.py @@ -13,6 +13,7 @@ class ProvidersEnum(StrEnum): copr = "copr" koji = "koji" url = "url" + container = "container" debug = "debug" @@ -21,4 +22,5 @@ class BuildIdTitleEnum(StrEnum): koji = "Koji build" packit = "Packit build" url = "URL" + container = "Container log" debug = "Debug output" diff --git a/backend/fetcher.py b/backend/fetcher.py index e57594d..578c057 100644 --- a/backend/fetcher.py +++ b/backend/fetcher.py @@ -61,6 +61,12 @@ def fetch_logs(self) -> list[dict[str, str]]: """ ... + +class RPMProvider(Provider): + """ + Is able to provide spec file on top of the logs. + """ + @abstractmethod def fetch_spec_file(self) -> dict[str, str]: """ @@ -72,7 +78,7 @@ def fetch_spec_file(self) -> dict[str, str]: ... -class CoprProvider(Provider): +class CoprProvider(RPMProvider): copr_url = "https://copr.fedorainfracloud.org" def __init__(self, build_id: int, chroot: str) -> None: @@ -130,7 +136,7 @@ def fetch_spec_file(self) -> dict[str, str]: return {"name": spec_name, "content": response.text} -class KojiProvider(Provider): +class KojiProvider(RPMProvider): koji_url = "https://koji.fedoraproject.org" logs_to_look_for = ["build.log", "root.log", "mock_output.log"] @@ -269,7 +275,7 @@ def fetch_spec_file(self) -> dict[str, str]: return spec_dict -class PackitProvider(Provider): +class PackitProvider(RPMProvider): """ The `packit_id` is hard to get. Open https://prod.packit.dev/api @@ -289,17 +295,19 @@ def __init__(self, packit_id: int) -> None: self.koji_url = f"{self.packit_api_url}/koji-builds/{self.packit_id}" def _get_correct_provider(self) -> CoprProvider | KojiProvider: - build = requests.get(self.copr_url).json() - if "error" not in build: + resp = requests.get(self.copr_url) + if resp.ok: + build = resp.json() return CoprProvider(build["build_id"], build["chroot"]) - build = requests.get(self.koji_url).json() - task_id = build["build_id"] - if "error" in build: + resp = requests.get(self.koji_url) + if not resp.ok: raise FetchError( f"Couldn't find any build logs for Packit ID #{self.packit_id}." ) + build = resp.json() + task_id = build["task_id"] koji_api_url = f"{KojiProvider.koji_url}/kojihub" koji_client = koji.ClientSession(koji_api_url) arch = koji_client.getTaskInfo(task_id, strict=True).get("arch") @@ -317,7 +325,7 @@ def fetch_spec_file(self) -> dict[str, str]: return self._get_correct_provider().fetch_spec_file() -class URLProvider(Provider): +class URLProvider(RPMProvider): def __init__(self, url: str) -> None: self.url = url @@ -327,7 +335,7 @@ def fetch_logs(self) -> list[dict[str, str]]: # also this will allow us to fetch spec files response = requests.get(self.url) response.raise_for_status() - if response.headers["Content-Type"] != "text/plain": + if "text/plain" not in response.headers["Content-Type"]: raise FetchError( "The URL must point to a raw text file. " f"This URL isn't: {self.url}" ) @@ -345,6 +353,31 @@ def fetch_spec_file(self) -> dict[str, str]: return {"name": "fake_spec_name.spec", "content": "fake spec file"} +class ContainerProvider(Provider): + """ + Fetching container logs only from URL for now + """ + + def __init__(self, url: str) -> None: + self.url = url + + @handle_errors + def fetch_logs(self) -> list[dict[str, str]]: + # TODO: c&p from url provider for now, integrate with containers better later on + response = requests.get(self.url) + response.raise_for_status() + if "text/plain" not in response.headers["Content-Type"]: + raise FetchError( + "The URL must point to a raw text file. " f"This URL isn't: {self.url}" + ) + return [ + { + "name": "Container log", + "content": response.text, + } + ] + + def fetch_debug_logs(): return [ { diff --git a/backend/schema.py b/backend/schema.py index 197e7ab..94e3996 100644 --- a/backend/schema.py +++ b/backend/schema.py @@ -1,20 +1,22 @@ from typing import Optional -from pydantic import AnyUrl, BaseModel +from pydantic import AnyUrl, BaseModel, root_validator from backend.constants import BuildIdTitleEnum -class LogSchema(BaseModel): - name: str - content: str +def _check_spec_container_are_exclusively_mutual(_, values): + spec_file = values.get("spec_file") + container_file = values.get("container_file") + if spec_file and container_file: + raise ValueError("You can't specify both spec file and container file") + return values -class SpecfileSchema(BaseModel): - # TODO: do we want to store spec_file separately - # in file or store content in one file? - # or the path means just its name? - # path: Path + +class NameContentSchema(BaseModel): + # TODO: do we want to store spec_file and container_file separately + # in file or store content in one file? Or the path means just its name? name: str content: str @@ -25,17 +27,25 @@ class ContributeResponseSchema(BaseModel): fetched data (logs, spec, ...) and are needed for user to give a feedback why build failed. """ + build_id: Optional[int] build_id_title: BuildIdTitleEnum build_url: AnyUrl - logs: list[LogSchema] - spec_file: SpecfileSchema + logs: list[NameContentSchema] + spec_file: Optional[NameContentSchema] = None + container_file: Optional[NameContentSchema] = None + + # validators + _normalize_spec_and_container_file = root_validator(pre=True, allow_reuse=True)( + _check_spec_container_are_exclusively_mutual + ) class SnippetSchema(BaseModel): """ Snippet for log, each log may have 0 - many snippets. """ + # LINE_FROM:CHAR_FROM-LINE_TO:CHAR_TO # log_part: constr(regex=r"^\d+:\d+-\d+:\d+$") start_index: int @@ -56,18 +66,25 @@ class SnippetSchema(BaseModel): # return self._splitter(False) -class FeedbackLogSchema(LogSchema): +class FeedbackLogSchema(NameContentSchema): """ Feedback from user per individual log with user's comment. """ + snippets: list[SnippetSchema] class _WithoutLogsSchema(BaseModel): username: Optional[str] - spec_file: SpecfileSchema fail_reason: str how_to_fix: str + spec_file: Optional[NameContentSchema] = None + container_file: Optional[NameContentSchema] = None + + # validators + _normalize_spec_and_container_file = root_validator(pre=True, allow_reuse=True)( + _check_spec_container_are_exclusively_mutual + ) class FeedbackInputSchema(_WithoutLogsSchema): @@ -75,6 +92,7 @@ class FeedbackInputSchema(_WithoutLogsSchema): Contains data from users with reasons why build failed. It is sent from FE and contains only inputs from user + spec and logs content. """ + logs: list[FeedbackLogSchema] @@ -83,18 +101,26 @@ class FeedbackSchema(_WithoutLogsSchema): This schema is the final structure as we decided to store our data in json file from users feedbacks. """ + logs: dict[str, FeedbackLogSchema] -def schema_inp_to_out(inp: FeedbackInputSchema) -> FeedbackSchema: +def schema_inp_to_out( + inp: FeedbackInputSchema, is_with_spec: bool = True +) -> FeedbackSchema: parsed_log_schema = {} for log_schema in inp.logs: parsed_log_schema[log_schema.name] = log_schema + if is_with_spec: + spec_or_container = {"spec_file": inp.spec_file} + else: + spec_or_container = {"container_file": inp.container_file} + return FeedbackSchema( username=inp.username, - spec_file=inp.spec_file, logs=parsed_log_schema, fail_reason=inp.fail_reason, how_to_fix=inp.how_to_fix, + **spec_or_container, ) diff --git a/backend/store.py b/backend/store.py index bc94c10..aeeabe0 100644 --- a/backend/store.py +++ b/backend/store.py @@ -30,7 +30,7 @@ def store(self, feedback_result: FeedbackSchema) -> None: timestamp_seconds = int(datetime.now().timestamp()) file_name = self.build_dir / f"{timestamp_seconds}.json" with open(file_name, "w") as fp: - json.dump(feedback_result.dict(), fp, indent=4) + json.dump(feedback_result.dict(exclude_unset=True), fp, indent=4) @staticmethod def _get_random_dir_from(dir_: Path) -> Path: diff --git a/frontend/src/app/contribute.cljs b/frontend/src/app/contribute.cljs index dc05584..407166e 100644 --- a/frontend/src/app/contribute.cljs +++ b/frontend/src/app/contribute.cljs @@ -28,6 +28,7 @@ snippets files spec + container error-description error-title backend-data @@ -64,6 +65,7 @@ (reset! build-id-title (:build_id_title data)) (reset! build-url (:build_url data)) (reset! spec (:spec_file data)) + (reset! container (:container_file data)) (reset! files @@ -85,7 +87,7 @@ [(instructions-item (not-empty @files) - (if (= @build-id-title "URL") + (if (contains? #{"URL" "Container log"} @build-id-title) [:<> (str "We fetched logs from ") [:a {:href @build-url} "this URL"]] diff --git a/frontend/src/app/contribute_atoms.cljs b/frontend/src/app/contribute_atoms.cljs index b01e96c..9ff4016 100644 --- a/frontend/src/app/contribute_atoms.cljs +++ b/frontend/src/app/contribute_atoms.cljs @@ -7,6 +7,7 @@ (def snippets (r/atom [])) (def files (r/atom nil)) (def spec (r/atom nil)) +(def container (r/atom nil)) (def error-description (r/atom nil)) (def error-title (r/atom nil)) diff --git a/frontend/src/app/contribute_events.cljs b/frontend/src/app/contribute_events.cljs index bc7e7c0..52bf3f1 100644 --- a/frontend/src/app/contribute_events.cljs +++ b/frontend/src/app/contribute_events.cljs @@ -23,6 +23,7 @@ snippets fas spec + container files]])) @@ -53,7 +54,8 @@ ;; We can't use @files here because they contain highlight ;; spans, HTML escaping, etc. (:logs @backend-data)) - :spec_file @spec}] + :spec_file @spec + :container_file @container}] (reset! status "submitting") (-> (fetch/post url {:accept :json :content-type :json :body body}) (.then (fn [resp] (-> resp :body (js->clj :keywordize-keys true)))) diff --git a/frontend/src/app/homepage.cljs b/frontend/src/app/homepage.cljs index e9b409c..0be92ad 100644 --- a/frontend/src/app/homepage.cljs +++ b/frontend/src/app/homepage.cljs @@ -18,12 +18,13 @@ (validate current-hash-atom input-values input-errors) (let [source (str/replace @current-hash-atom "#" "") params (match @current-hash-atom - "#copr" [(get @input-values :copr-build-id) - (get @input-values :copr-chroot)] - "#packit" [(get @input-values :packit-id)] - "#koji" [(get @input-values :koji-build-id) - (get @input-values :koji-arch)] - "#url" [(js/btoa (get @input-values :url))]) + "#copr" [(get @input-values :copr-build-id) + (get @input-values :copr-chroot)] + "#packit" [(get @input-values :packit-id)] + "#koji" [(get @input-values :koji-build-id) + (get @input-values :koji-arch)] + "#url" [(js/btoa (get @input-values :url))] + "#container" [(js/btoa (get @input-values :url))]) url (str/join "/" (concat ["/contribute" source] (map str/trim params)))] (when (empty? @input-errors) (set! (.-href (.-location js/window)) url)))) @@ -51,7 +52,8 @@ (render-navigation-item "Copr" "#copr") (render-navigation-item "Koji" "#koji") (render-navigation-item "Packit" "#packit") - (render-navigation-item "URL" "#url")]]) + (render-navigation-item "URL" "#url") + (render-navigation-item "Container" "#container")]]) (defn render-card [provider url title img text inputs] [:div {:class "card-body"} @@ -122,19 +124,29 @@ (render-card nil nil - "Submit logs from URL" + "Submit RPM logs from URL" "img/url-icon.png" (str/join "" ["Paste an URL to a log file, or a build in some build system. " "If recognized, we will fetch and display all relevant logs."]) [(input "url" "https://paste.centos.org/view/raw/5ba21754")])) +(defn render-container-card [] + (render-card + nil + nil + "Submit container logs from URL" + "img/url-icon.png" + "Paste an URL to a raw container log file." + [(input "url" "https://paste.centos.org/view/raw/5ba21754")])) + (defn render-cards [] (match @current-hash-atom - "#copr" (render-copr-card) - "#packit" (render-packit-card) - "#koji" (render-koji-card) - "#url" (render-url-card) - :else (render-copr-card))) + "#copr" (render-copr-card) + "#packit" (render-packit-card) + "#koji" (render-koji-card) + "#url" (render-url-card) + "#container" (render-container-card) + :else (render-copr-card))) (defn homepage [] (reset! current-hash-atom diff --git a/frontend/src/app/homepage_validation.cljs b/frontend/src/app/homepage_validation.cljs index 0fa9eab..0f56c61 100644 --- a/frontend/src/app/homepage_validation.cljs +++ b/frontend/src/app/homepage_validation.cljs @@ -42,6 +42,11 @@ (swap! input-errors conj "koji-arch"))) "#url" + (do + (when (empty? (get @input-values :url)) + (swap! input-errors conj "url"))) + + "#container" (do (when (empty? (get @input-values :url)) (swap! input-errors conj "url")))))