From 8df7baa89a3446a4a3035f73bc3dc903c3567b13 Mon Sep 17 00:00:00 2001 From: Honza Javorek Date: Wed, 17 Jan 2024 17:03:27 +0100 Subject: [PATCH] add weworkremotely --- juniorguru_plucker/jobs_startupjobs/spider.py | 1 - .../jobs_weworkremotely/.actor/actor.json | 11 + .../jobs_weworkremotely/__init__.py | 0 .../jobs_weworkremotely/spider.py | 95 ++++++++ poetry.lock | 216 +++++++++++++++++- pyproject.toml | 5 +- tests/jobs_weworkremotely/job.html | 135 +++++++++++ .../job_json_decode_error.html | 120 ++++++++++ tests/jobs_weworkremotely/job_no_image.html | 98 ++++++++ .../remote-programming-jobs.rss | 183 +++++++++++++++ tests/jobs_weworkremotely/test_spider.py | 103 +++++++++ 11 files changed, 964 insertions(+), 3 deletions(-) create mode 100644 juniorguru_plucker/jobs_weworkremotely/.actor/actor.json create mode 100644 juniorguru_plucker/jobs_weworkremotely/__init__.py create mode 100644 juniorguru_plucker/jobs_weworkremotely/spider.py create mode 100644 tests/jobs_weworkremotely/job.html create mode 100644 tests/jobs_weworkremotely/job_json_decode_error.html create mode 100644 tests/jobs_weworkremotely/job_no_image.html create mode 100644 tests/jobs_weworkremotely/remote-programming-jobs.rss create mode 100644 tests/jobs_weworkremotely/test_spider.py diff --git a/juniorguru_plucker/jobs_startupjobs/spider.py b/juniorguru_plucker/jobs_startupjobs/spider.py index 00d3f35..1478eb3 100644 --- a/juniorguru_plucker/jobs_startupjobs/spider.py +++ b/juniorguru_plucker/jobs_startupjobs/spider.py @@ -1,5 +1,4 @@ import html -from datetime import datetime from typing import Generator from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst diff --git a/juniorguru_plucker/jobs_weworkremotely/.actor/actor.json b/juniorguru_plucker/jobs_weworkremotely/.actor/actor.json new file mode 100644 index 0000000..700204b --- /dev/null +++ b/juniorguru_plucker/jobs_weworkremotely/.actor/actor.json @@ -0,0 +1,11 @@ +{ + "actorSpecification": 1, + "name": "jobs-weworkremotely", + "title": "jobs-weworkremotely", + "version": "0.0", + "dockerfile": "../../../Dockerfile", + "dockerContextDir": "../../../", + "storages": { + "dataset": "../../schemas/jobSchema.json" + } +} diff --git a/juniorguru_plucker/jobs_weworkremotely/__init__.py b/juniorguru_plucker/jobs_weworkremotely/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/juniorguru_plucker/jobs_weworkremotely/spider.py b/juniorguru_plucker/jobs_weworkremotely/spider.py new file mode 100644 index 0000000..d4b875e --- /dev/null +++ b/juniorguru_plucker/jobs_weworkremotely/spider.py @@ -0,0 +1,95 @@ +import html +import json +import time +from datetime import date, datetime + +import extruct +import feedparser +from itemloaders.processors import Identity, MapCompose, TakeFirst +from lxml import etree +from scrapy import Spider as BaseSpider +from scrapy.loader import ItemLoader + +from juniorguru_plucker.items import Job +from juniorguru_plucker.processors import absolute_url + + +class Spider(BaseSpider): + name = "jobs-weworkremotely" + start_urls = [ + "https://weworkremotely.com/categories/remote-devops-sysadmin-jobs.rss", + "https://weworkremotely.com/categories/remote-programming-jobs.rss", + ] + + def parse(self, response): + for entry in feedparser.parse(response.text).entries: + feed_data = dict( + title=entry.title, + first_seen_on=parse_struct_time(entry.published_parsed), + company_logo_urls=[ + c["url"] for c in getattr(entry, "media_content", []) + ], + description_html=entry.summary, + remote=True, + source_urls=response.url, + ) + yield response.follow( + entry.link, callback=self.parse_job, cb_kwargs=dict(feed_data=feed_data) + ) + + def parse_job(self, response, feed_data): + loader = Loader(item=Job(), response=response) + loader.add_value("url", response.url) + + for key, value in feed_data.items(): + loader.add_value(key, value) + + try: + data = extract_job_posting(response.text, response.url) + except (ValueError, json.JSONDecodeError, etree.ParserError): + pass + else: + loader.add_value("source", self.name) + loader.add_value("source_urls", response.url) + loader.add_value("title", data["title"]) + loader.add_value("first_seen_on", data["datePosted"]) + loader.add_value("description_html", html.unescape(data["description"])) + loader.add_value("company_logo_urls", data.get("image")) + loader.add_value("employment_types", [data["employmentType"]]) + loader.add_value("company_name", data["hiringOrganization"]["name"]) + loader.add_value("company_url", data["hiringOrganization"]["sameAs"]) + loader.add_value("locations_raw", data["hiringOrganization"]["address"]) + yield loader.load_item() + + +def parse_struct_time(struct_time): + if struct_time: + return datetime.fromtimestamp(time.mktime(struct_time)).date() + + +def parse_date(value: str | None) -> date | None: + if value: + return date.fromisoformat(value[:10]) + + +def extract_job_posting(html_string, base_url): + data = extruct.extract(html_string, base_url, syntaxes=["json-ld"]) + try: + return [ + data_item + for data_item in data["json-ld"] + if data_item["@type"] == "JobPosting" + ][0] + except IndexError: + raise ValueError("json-ld provided no job postings") + + +class Loader(ItemLoader): + default_input_processor = MapCompose(str.strip) + default_output_processor = TakeFirst() + company_url_in = MapCompose(absolute_url) + first_seen_on_in = MapCompose(parse_date) + company_logo_urls_out = Identity() + remote_in = MapCompose(bool) + locations_raw_out = Identity() + source_urls_out = Identity() diff --git a/poetry.lock b/poetry.lock index 7050788..c15d904 100644 --- a/poetry.lock +++ b/poetry.lock @@ -141,6 +141,24 @@ six = "*" [package.extras] visualize = ["Twisted (>=16.1.1)", "graphviz (>0.5.1)"] +[[package]] +name = "beautifulsoup4" +version = "4.12.2" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, + {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "certifi" version = "2023.11.17" @@ -407,6 +425,44 @@ files = [ {file = "cssselect-1.2.0.tar.gz", hash = "sha256:666b19839cfaddb9ce9d36bfe4c969132c647b92fc9088c4e23f786b30f1b3dc"}, ] +[[package]] +name = "extruct" +version = "0.16.0" +description = "Extract embedded metadata from HTML markup" +optional = false +python-versions = "*" +files = [ + {file = "extruct-0.16.0-py2.py3-none-any.whl", hash = "sha256:2499ea9e7d22744745ca708acee9542a4aa231871620c4f65f869a1286e64aa8"}, + {file = "extruct-0.16.0.tar.gz", hash = "sha256:d09cb3d86d149a276b277b3bd45b2b867ef3ec78bed9cd58ee0f2ae01ae670c4"}, +] + +[package.dependencies] +html-text = ">=0.5.1" +jstyleson = "*" +lxml = "*" +mf2py = "*" +pyrdfa3 = "*" +rdflib = {version = ">=6.0.0", markers = "python_version >= \"3.7\""} +six = "*" +w3lib = "*" + +[package.extras] +cli = ["requests"] + +[[package]] +name = "feedparser" +version = "6.0.11" +description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" +optional = false +python-versions = ">=3.6" +files = [ + {file = "feedparser-6.0.11-py3-none-any.whl", hash = "sha256:0be7ee7b395572b19ebeb1d6aafb0028dee11169f1c934e0ed67d54992f4ad45"}, + {file = "feedparser-6.0.11.tar.gz", hash = "sha256:c9d0407b64c6f2a065d0ebb292c2b35c01050cc0dc33757461aaabdc4c4184d5"}, +] + +[package.dependencies] +sgmllib3k = "*" + [[package]] name = "filelock" version = "3.13.1" @@ -434,6 +490,41 @@ files = [ {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] +[[package]] +name = "html-text" +version = "0.5.2" +description = "Extract text from HTML" +optional = false +python-versions = "*" +files = [ + {file = "html_text-0.5.2-py2.py3-none-any.whl", hash = "sha256:3f1e063f05eddf3e099a88f0440219c55fdc01c44f1291fe59c66e5228d7fc56"}, + {file = "html_text-0.5.2.tar.gz", hash = "sha256:afd61bbb70651d494a8c32670a29b9140492eccc9690109857beae41c3093ded"}, +] + +[package.dependencies] +lxml = "*" + +[[package]] +name = "html5lib" +version = "1.1" +description = "HTML parser based on the WHATWG HTML specification" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"}, + {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"}, +] + +[package.dependencies] +six = ">=1.9" +webencodings = "*" + +[package.extras] +all = ["chardet (>=2.2)", "genshi", "lxml"] +chardet = ["chardet (>=2.2)"] +genshi = ["genshi"] +lxml = ["lxml"] + [[package]] name = "httpcore" version = "1.0.2" @@ -530,6 +621,20 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + [[package]] name = "itemadapter" version = "0.8.0" @@ -569,6 +674,16 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "jstyleson" +version = "0.0.2" +description = "Library to parse JSON with js-style comments." +optional = false +python-versions = "*" +files = [ + {file = "jstyleson-0.0.2.tar.gz", hash = "sha256:680003f3b15a2959e4e6a351f3b858e3c07dd3e073a0d54954e34d8ea5e1308e"}, +] + [[package]] name = "lxml" version = "5.1.0" @@ -677,6 +792,22 @@ files = [ docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.5)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] testing = ["coverage", "pyyaml"] +[[package]] +name = "mf2py" +version = "2.0.1" +description = "Microformats parser" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mf2py-2.0.1-py3-none-any.whl", hash = "sha256:092806e17f1a93db4aafa5e8d3c4124b5e42cd89027e2db48a5248ef4eabde03"}, + {file = "mf2py-2.0.1.tar.gz", hash = "sha256:1380924633413b8d72e704b5c86b4382c4b1371699edecc907b01cd21138d7cd"}, +] + +[package.dependencies] +beautifulsoup4 = ">=4.11.1,<5.0.0" +html5lib = ">=1.1,<2.0" +requests = ">=2.28.2,<3.0.0" + [[package]] name = "nest-asyncio" version = "1.5.9" @@ -856,6 +987,20 @@ cryptography = ">=41.0.5,<42" docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"] test = ["flaky", "pretend", "pytest (>=3.0.1)"] +[[package]] +name = "pyparsing" +version = "3.1.1" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.6.8" +files = [ + {file = "pyparsing-3.1.1-py3-none-any.whl", hash = "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb"}, + {file = "pyparsing-3.1.1.tar.gz", hash = "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "pypydispatcher" version = "2.1.2" @@ -866,6 +1011,22 @@ files = [ {file = "PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2"}, ] +[[package]] +name = "pyrdfa3" +version = "3.6.2" +description = "pyRdfa distiller/parser library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyRdfa3-3.6.2-py3-none-any.whl", hash = "sha256:290c2fa966ddd1b45ac94a727da144f5a233ed58c63c370e3d68e6d00b0dee5d"}, + {file = "pyRdfa3-3.6.2.tar.gz", hash = "sha256:73681dab957f60901696767388b956a5769c730bc451da6ffb2f0e36f18314c2"}, +] + +[package.dependencies] +html5lib = ">=1.1" +rdflib = ">=6.1.1" +requests = ">=2.25.1" + [[package]] name = "pytest" version = "7.4.4" @@ -911,6 +1072,27 @@ files = [ {file = "queuelib-1.6.2.tar.gz", hash = "sha256:4b207267f2642a8699a1f806045c56eb7ad1a85a10c0e249884580d139c2fcd2"}, ] +[[package]] +name = "rdflib" +version = "7.0.0" +description = "RDFLib is a Python library for working with RDF, a simple yet powerful language for representing information." +optional = false +python-versions = ">=3.8.1,<4.0.0" +files = [ + {file = "rdflib-7.0.0-py3-none-any.whl", hash = "sha256:0438920912a642c866a513de6fe8a0001bd86ef975057d6962c79ce4771687cd"}, + {file = "rdflib-7.0.0.tar.gz", hash = "sha256:9995eb8569428059b8c1affd26b25eac510d64f5043d9ce8c84e0d0036e995ae"}, +] + +[package.dependencies] +isodate = ">=0.6.0,<0.7.0" +pyparsing = ">=2.1.0,<4" + +[package.extras] +berkeleydb = ["berkeleydb (>=18.1.0,<19.0.0)"] +html = ["html5lib (>=1.0,<2.0)"] +lxml = ["lxml (>=4.3.0,<5.0.0)"] +networkx = ["networkx (>=2.0.0,<3.0.0)"] + [[package]] name = "requests" version = "2.31.0" @@ -1044,6 +1226,16 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "sgmllib3k" +version = "1.0.0" +description = "Py3k port of sgmllib." +optional = false +python-versions = "*" +files = [ + {file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"}, +] + [[package]] name = "six" version = "1.16.0" @@ -1091,6 +1283,17 @@ files = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + [[package]] name = "tldextract" version = "5.1.1" @@ -1215,6 +1418,17 @@ files = [ {file = "w3lib-2.1.2.tar.gz", hash = "sha256:ed5b74e997eea2abe3c1321f916e344144ee8e9072a6f33463ee8e57f858a4b1"}, ] +[[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" +optional = false +python-versions = "*" +files = [ + {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, + {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, +] + [[package]] name = "websockets" version = "12.0" @@ -1352,4 +1566,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "3.11.*" -content-hash = "7d5335b9ad9ff4c4aafc4bc7de1eb5474ba51caa4ad158eeea6fe9fc294d8172" +content-hash = "8a62d8898d607e699204ad76514efa591a80583a4d480eaa27da89743cb64be7" diff --git a/pyproject.toml b/pyproject.toml index d3362b1..03c4071 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,8 @@ nest-asyncio = "1.5.9" scrapy = "2.11.0" click = "8.1.7" markdown = "3.5.2" +feedparser = "6.0.11" +extruct = "0.16.0" [tool.poetry.group.dev.dependencies] pytest = "7.4.4" @@ -27,7 +29,8 @@ python_files = "test_*.py" testpaths = "tests" addopts = "--import-mode=importlib --ff --ruff --ruff-format" filterwarnings = [ - "ignore:'cgi' is deprecated:DeprecationWarning", + "ignore:'cgi' is deprecated:DeprecationWarning", # scrapy (twisted) + "ignore:invalid escape sequence:DeprecationWarning", # extruct ] [tool.ruff] diff --git a/tests/jobs_weworkremotely/job.html b/tests/jobs_weworkremotely/job.html new file mode 100644 index 0000000..18e0080 --- /dev/null +++ b/tests/jobs_weworkremotely/job.html @@ -0,0 +1,135 @@ +Bluelight Consulting: DevOps Engineer, Kubernetes, AWS/GCP

See more DevOps and Sysadmin jobs→

← Back to all jobs

Posted

DevOps Engineer, Kubernetes, AWS/GCP

Bluelight Consulting

View all jobs →

Full-TimeDevOps and SysadminUSA Only

Bluelight Consulting

Sacramento, CA

View website

Jobs posted: 1

Apply for this position
View all jobs→

DESCRIPTION
DevOps-as-a-Service provider, Bluelight Consulting, is seeking a DevOps Engineer to join its growing team.

We specialize in working with Kubernetes for cloud-native projects in GCP, AWS, and Serverless. Our engineers enjoy wearing many technology hats of a small boutique firm without corporate headaches. We work Monday-Friday, with flexible hours.

Our engineers love making decisions in real-time. They work hands-on with technologies like GKE/EKS, Terraform, Helm, CI/CD, and more.




*We are hiring for multiple DevOps Engineer roles from Staff level to Lead Engineer. All roles are 100% remote.





REQUIREMENTS
  • Cloud Engineering (cloud computing) experience with AWS and/or GCP to include load balancing
  • Infrastructure as a code, specifically Terraform / Pulumi / Cloudformation experience
  • Design and maintain CI/CD process and tools (specifically CircleCI, GitLab, Jenkins)
  • In-depth experience with the orchestration tools, especially Kubernetes
  • In-depth experience with the YAML config management tools, especially Helm & Ansible
  • Testing, code review, good communication skills

Nice to have qualifications:
  • AWS Professional Certificates
  • Kubernetes Certificates
  • Google Cloud Certificates

Duties:
  • Architect and integrate new applications along with occasional legacy applications
  • Build and maintain, update tool for deployment, monitoring, operations, and security
  • Troubleshoot and resolve issues in dev, testing, and production environments
  • Structure and maintain software config management systems
  • Scale for demand
  • Work with high growth, technology clients
  • Develop and improve operational practices and procedures
  • Suggest improvements and enhancements
  • Bring new features and services into production





BENEFITS
  • Competitive salary and bonuses, including performance-based salary increases
  • Generous paid-time-off policy
  • Health Coverage
  • Flexible working hours
  • Work remotely
  • Bonuses for each certification
  • Continuing education, training, conferences
  • Company-sponsored coursework, exams, and certifications
Apply for this position
  • Share this job:


Help us maintain the quality of jobs posted on We Work Remotely.

Is this job not remote?

Let us know!

Apply for this Position

Please ensure you meet geographic and skills requirements before applying.

  • Share this job:


Help us maintain the quality of jobs posted on We Work Remotely.

Is this job not remote?

Let us know!
diff --git a/tests/jobs_weworkremotely/job_json_decode_error.html b/tests/jobs_weworkremotely/job_json_decode_error.html new file mode 100644 index 0000000..686fd4e --- /dev/null +++ b/tests/jobs_weworkremotely/job_json_decode_error.html @@ -0,0 +1,120 @@ +ElevenYellow Pte Ltd: Solidity Engineer




See more Programming jobs→

← Back to all jobs

Posted

Solidity Engineer

ElevenYellow Pte Ltd

View all jobs →

Full-TimeProgrammingAnywhere (100% Remote) Only

ElevenYellow Pte Ltd

ElevenYellow Pte Ltd

View website

Jobs posted: 3

Apply for this position
View all jobs→

Solidity Engineer

Note: We’re currently looking for candidates in any timezone! Regardless of where you are, we’d be happy to see you apply, especially if you feel you’re a great fit. This is a remote, full-time position.

Who we are:

Elevenyellow is a multidisciplinary group of people building interesting projects in the crypto space and beyond. From infrastructure, to wallets, to NFT art... our team of 40 has been playing around the space before it was cool. We’re so hipster that we were a 100% remote company in 2013 just in case covid hit ¯\_(ツ)_/¯

Who we’re looking for:

We are looking for a talented, motivated blockchain engineer, specialized in Ethereum (Solidity). We have an upcoming project around NFTs and AI-based art that’s going to blow your mind. And from there, there will be many more things coming up.

You are a great fit if you have a demonstrated record of:

  • Finding solutions to tough problems by designing incentive mechanisms, protocol optimisations and governance mechanics in a fast moving DeFi ecosystem
  • Developing, testing and deploying smart contracts
  • Developing high-quality code that is maintainable, performant and accessible
  • Continuously exploring opportunities in DeFi
  • Enjoying working in a team of self-motivated people


What you bring to our engineering team:

  • Deep understanding of DeFi protocols
  • Deep understanding of Solidity
  • Creative problem solving skills with analytical mindset
  • Good understanding of continuous integration tools and blockchain management tools


We’re serious about…


Product Ownership


We believe that when you build a product, it’s your baby (conveniently enough some of our developers are dads so they share this mentality). And well… babies need nurturing!

We’re not looking for somebody to crap out an MVP, we’re looking for someone who wants to build the next best thing with us.

Design

That said, we take pride in our projects. While we launch a lot of them, we like to build in speed and in style. Projects we deem successful require a solid design that’s not just an ugly, out the box template.

Speed

Hopefully you like moving QUICKLY. We don’t spend weeks mocking up things. Our developers are quick and talented, meaning we get projects out in the open fast.


We’re not trying to convince you… but we have our perks


Work remotely

We’re very experienced in working remotely. Feel free to pick up your laptop and work wherever you want, whenever you want. Just make sure you have a good internet connection and we’re good :)

Take vacay

We know what it feels like to work “too much” and our way of work isn’t meant to be stressful. Time off is the remedy to a lot of things, and we encourage you to take time off work. Three weeks is the minimum.

Annual Retreats

We do yearly, week-long retreats and we’d love if you joined us (we pay your way, don’t worry). It’s important to meet each other in person and we value time spent with one another (whether that’s on Slack or face-to-face). We’ve met up all over Europe and islands across South East Asia. All in good fun. Last year we went to Gran Canaria, and this year we’ve been to the south of Italy!

We’re a laid back bunch

While team retreats sound awesome (and they are), our company ethos is what we value the most. Life is all about enjoyment, and work should be too. We’re all laid back and understand the human side of work and play.


Oh and these things too

  • Competitive Salary (remuneration depends on experience)
  • Health Insurance Coverage (monthly, set budget)
  • Self Improvement Budget (monthly, set budget)
  • Gym Membership (Monthly, set budget)

Now, here’s what you need to do:

  1. Send an email to enter[a-t]elevenyellow.com
  2. Mention why you think you’re a good fit for us
  3. Describe your career highlights (highs and lows, best/worst projects, etc)
  4. Add your portfolio / things that really define your work
  5. No CVs/Linkedin, we don’t know how to read those.


We believe applications should be human and conversational. We’d mostly appreciate if you told us who you were and what you wanted to do in life, both personally and professionally.

Of course, please only disclose what you’re comfortable with. And not to forget! We’re an equal opportunity employer and we encourage everyone to apply. We respect genders, ethnicities, boundaries, friendships, and the human aspect of work.

Apply for this position
  • Share this job:


Help us maintain the quality of jobs posted on We Work Remotely.

Is this job not remote?

Let us know!

Apply for this position

Please ensure you meet geographic and skills requirements before applying.

  • Share this job:


Help us maintain the quality of jobs posted on We Work Remotely.

Is this job not remote?

Let us know!
\ No newline at end of file diff --git a/tests/jobs_weworkremotely/job_no_image.html b/tests/jobs_weworkremotely/job_no_image.html new file mode 100644 index 0000000..9bb7758 --- /dev/null +++ b/tests/jobs_weworkremotely/job_no_image.html @@ -0,0 +1,98 @@ +Stealth Co: Devops Engineer

See more DevOps and Sysadmin jobs→

← Back to all jobs

Posted

Devops Engineer

Stealth Co

View all jobs →

Full-TimeDevOps and SysadminAnywhere (100% Remote) Only

Stealth Co

San Francisco

View website

Jobs posted: 1

Apply for this position
View all jobs→

We are a stealth software company looking for an experienced Devops Engineer to help us grow and administer our platform and infrastructure.

We prefer someone to be available during US business hours and with a work schedule that lines up with ours.
Must understand and speak English.
USA, Canada, Eastern Europe, Western Europe, Australia and NZ encouraged to apply.

Required skills:

Intermediate to advanced knowledge of Ubuntu Linux;

Understanding of networking concepts such as VLANs, mulltihoming, bridging, and link aggregation;

Familiarity with Puppet and Ruby;

Experience working in a physical data center environment.


Recommended skills:

Facility with DNS (bind 9)

Experience with provisioning automation

Experience with monitoring tools such as Nagios and Cacti;

Scripting in Bash and Ruby

Familiarity with Git & GitHub

Understanding of Nginx & Apache;

Knowledge of VirtualBox & Vagrant.

To apply, please email your resume and rate to the email on the apply button.


Good luck and thank you!




Apply for this position
  • Share this job:


Help us maintain the quality of jobs posted on We Work Remotely.

Is this job not remote?

Let us know!

Apply for this position

Please ensure you meet geographic and skills requirements before applying.

  • Share this job:


Help us maintain the quality of jobs posted on We Work Remotely.

Is this job not remote?

Let us know!
diff --git a/tests/jobs_weworkremotely/remote-programming-jobs.rss b/tests/jobs_weworkremotely/remote-programming-jobs.rss new file mode 100644 index 0000000..124d1a5 --- /dev/null +++ b/tests/jobs_weworkremotely/remote-programming-jobs.rss @@ -0,0 +1,183 @@ + + + + We Work Remotely: Programming Jobs + https://weworkremotely.com/categories/remote-programming-jobs.rss + We Work Remotely: Programming Jobs + en-US + 60 + + 10up: Senior UI Engineer + Anywhere (100% Remote) Only + Programming + + +<p> + <strong>Headquarters:</strong> Remote + <br /><strong>URL:</strong> <a href="https://10up.com">https://10up.com</a> +</p> + +<div>Be a part of <strong>something great.<br></strong><br> +</div><div><strong>Responsibilities</strong></div><ul> +<li>Interpret designs and transform them into a polished rich cross-platform/device web experience.</li> +<li>Write modular and performant HTML and CSS.</li> +<li>Ability to make smart design decisions. Occasionally filling in gaps left by designers e.g. interpreting how a layout should be presented on small screens.</li> +<li>Ability to rapidly prototype interfaces to finalize an experience in the browser as well as build long term scalable, maintainable, and performant production code.</li> +<li>Visualize how a static design will be coded. Proactively flag challenging areas and offer alternatives, as well as identifying areas for greater code reuse.</li> +<li>Ability to estimate work for a project and modularize goals into clearly defined, executable tasks.</li> +<li>Ability to lead the front-end strategy for a project and make impactful decisions about the methodologies recommended to a client.</li> +<li>Write JavaScript using industry best practices as well as React components. Ability to use modern JavaScript build tools like Webpack.</li> +<li>Effectively collaborate with designers throughout the design and engineering process.</li> +</ul><div><strong><br>Requirements</strong></div><ul> +<li>At least five years experience with UI engineering, web development, web engineering, or a similar field.</li> +<li>Strong understanding of HTML and CSS including cross-browser compatibility and performance.</li> +<li>Ability to write modern and performant JavaScript using latest technologies.</li> +<li>Experience with the React JavaScript library.</li> +<li>Experience using Git.</li> +<li>Proven ability to transform a static comp into a website from the ground up.</li> +<li>Understanding of ad implementations in a responsive environment.</li> +<li>Experience in task runners and bundlers like Webpack or Gulp.</li> +<li>Experience with coded style guides and creating highly componentized systems.</li> +<li>Experience with coding PHP, ideally for WordPress as a platform.</li> +<li>Being collaborative, self-motivated, and an "always-learning" person, excited to create great web experiences.</li> +</ul><div><strong><br>Nice to have</strong></div><ul> +<li>Experience with Sketch or Design Software.</li> +<li>Advanced understanding of as implementations in a responsive environment.</li> +<li>Moderate PHP experience.</li> +<li>Experience with Gutenberg and building out custom blocks.</li> +<li>Design experience or formal design education.</li> +<li>Remote working experience appreciated.</li> +<li>Experience in mentoring and leading a team.</li> +<li>Experience and expertise with third party integrations and client-side APIs.</li> +<li>Strong ability to communicate with client stakeholders, both technical and non-technical, about the impact of decisions around markup approach, cross-platform responsive interactions, performance, and topics like accessibility and internationalization.</li> +<li>Understand the high-level concepts behind modern JavaScript frameworks and can architect and execute project-work as needed.</li> +<li>Code-based portfolio. Link us to your GitHub account, upload a .zip, or both (source files are helpful to get a sense of your code structure and file organization).</li> +<li>Understanding of WCAG accessibility compliance. Ability to implement accessibility standards as well as audit existing websites.</li> +</ul> + +<p><strong>To apply:</strong> <a href="https://weworkremotely.com/remote-jobs/10up-senior-ui-engineer">https://weworkremotely.com/remote-jobs/10up-senior-ui-engineer</a></p> + + Wed, 28 Oct 2020 21:35:29 +0000 + https://weworkremotely.com/remote-jobs/10up-senior-ui-engineer + https://weworkremotely.com/remote-jobs/10up-senior-ui-engineer + + + + Flightradar24: Senior C++ Developer + Europe Only + Programming + <img src="https://we-work-remotely.imgix.net/logos/0017/2713/logo.gif?ixlib=rails-4.0.0&w=50&h=50&dpr=2&fit=fill&auto=compress" /> + +<p> + <strong>Headquarters:</strong> Stockholm, Sweden + <br /><strong>URL:</strong> <a href="https://www.flightradar24.com/">https://www.flightradar24.com/</a> +</p> + +<div>Flightradar24 is looking for experienced, well-rounded C++ and Python developers. You will play an important role in building and improving our back-end systems, processing very high volumes of aviation data like flight positions each day.<br><br> +</div><div>Your work will improve the experience of millions of daily Flightradar24 users, as well as enable our business and enterprise customers to effectively integrate our aviation data and services with their businesses.<br><strong><br>What you’ll do</strong> +</div><ul> +<li>Develop our back-end systems using modern C++ and Python on a Linux platform, using open source tools like RabbitMQ, Redis, and MySQL, as well as cloud services</li> +<li>Improve our flight tracking coverage with semi-embedded C++ development for our global network of Raspberry Pi-based ADS-B receivers (approximately 20,000 devices)</li> +<li>Design and implement big data streaming, ingestion, and event processing using both cloud and on-premise SQL and NoSQL systems</li> +<li>Expand our flight event detection logic and find new ways to use our data</li> +<li>Improve robustness of our systems using cloud infrastructure like AWS and Azure, and tools like Terraform, Ansible, Docker, and Kubernetes</li> +<li>Apply analytic and algorithmic skills to solve software design and aviation tracking challenges</li> +</ul><div><br></div><div><strong>Who you are</strong></div><ul> +<li>Experienced software engineer with at least 4 years of professional development, ideally in online/web services environments and with similar tech stacks</li> +<li>Experience with modern C++ '11-'20, STL and Boost, Python 3, as well as an understanding of data structures, algorithms and their use cases and efficiency</li> +<li>Passionate about development best practices and quality efforts, such as test-driven development, unit testing, code reviews, continuous integration, etc.</li> +<li>You know how to design simple, performant, testable, and maintainable software</li> +<li>You love what you do and are passionate about code and technology</li> +<li>You have a university degree in computer science or similar</li> +<li>You have strong written and spoken English – Swedish is not needed for this role</li> +<li>If you have experience with aviation data standards including ADS-B, that’s a big plus</li> +</ul><div><br></div><div> +<strong>About Flightradar24</strong><br>With over 2 million daily users, Flightradar24 is the world’s most popular flight tracking service and our apps regularly top the App Store and Google Play charts. We also offer a wide range of commercial services and customers include many of the largest names in aviation. <br><br>We're constantly adding new services and improving existing products. To help us meet those challenges, we're looking for creative, collaborative and tech-savvy applicants to join us. <br><br><br> +</div> + +<p><strong>To apply:</strong> <a href="https://weworkremotely.com/remote-jobs/flightradar24-senior-cpp-developer">https://weworkremotely.com/remote-jobs/flightradar24-senior-cpp-developer</a></p> + + Wed, 28 Oct 2020 20:10:05 +0000 + https://weworkremotely.com/remote-jobs/flightradar24-senior-cpp-developer + https://weworkremotely.com/remote-jobs/flightradar24-senior-cpp-developer + + + + BBK Worldwide: Software Developer + North America Only + Programming + <img src="https://we-work-remotely.imgix.net/logos/0017/2693/logo.gif?ixlib=rails-4.0.0&w=50&h=50&dpr=2&fit=fill&auto=compress" /> + +<p> + <strong>Headquarters:</strong> Needham, Massachusetts, United States + <br /><strong>URL:</strong> <a href="http://bbkworldwide.com">http://bbkworldwide.com</a> +</p> + +<div><strong>Software Developer</strong></div><div><em>Contribute to our industry-leading product and enjoy a convivial, fun, and welcoming corporate environment.</em></div><div> </div><div>BBK Worldwide, a first-of-its-kind marketing consultancy specializing in global patient recruitment for the clinical R&amp;D segments of the pharmaceutical, biotechnology, and medical device industries, is seeking a professionally minded individual looking to further a career in software development. </div><div> </div><div>The ideal candidate is operational, organized, efficient, talented, and has natural communications and interpersonal skills. He or she also enjoys a team-based, collaborative approach to performing job responsibilities, as well as the ability to work independently at a fast pace when necessary. Ability to rapidly incorporate direction and feedback is a must, as is a degree in computer science or a related field. Experience with or knowledge of software for the healthcare industry is a plus. More than just executing direction, we’re looking for the person who will provide insight and enhancement to the product as they work.  </div><div> </div><div><strong>Key responsibilities: </strong></div><div>·       Confer with systems analysts, engineers, programmers and others to design system and to obtain information on project limitations and capabilities, performance requirements and interfaces. </div><div>·       Execute front end development with ability to handle entire back-end code. </div><div>·       Analyze user needs and software requirements to determine feasibility of design within time and cost constraints.</div><div>·       Consult with customers about software system design and maintenance.</div><div>·       Determine system performance standards.</div><div>·       Developing, designing, or creating new applications, ideas, relationships, systems, or products, including artistic contributions. </div><div>·       Modify software programs to improve performance. </div><div>·       Handle excel data imports from SFTP location. </div><div>·       Assess database performance. </div><div>·       Write technical documentation. </div><div> </div><div><strong>Must have experience with:</strong></div><div>·       AngularJS 1.6 and Kendo UI, Asp.Net MVC, ASP.Net Web API and C#. ASP.NET 4.0 +, VB.Net, WCF, Telerik ASP.Net Controls.</div><div>·       Responsive web development; HTML 5, CSS 3.0, Bootstrap. (3 to 5 Years)</div><div>·       Hands on experience on SQL Server 2014 and 2016.</div><div>·       Ability to write complex SQL procedures, views, queries and able to perform performance optimization when required. (5+ years)</div><div>·       Integrating third party APIs. (5+ years)</div><div>·       SQL Server Integration Service (SSIS) Packages</div><div>·       Team foundation server (TFS) 2012 and 2017 source code management </div><div>·       IIS 7.0, 7.5, 8.0 and website configurations in webfarm / load balancing environment </div><div>·       Working knowledge of Azure is plus </div><div>·       Ability to work in Scrum / Agile model and be able to program both quickly and accurately</div><div>·       Ability to implement secure coding practices using OWASP and Microsoft guidelines</div><div>·       Ability to implement DevOps methodologies and tools collaboration and communication of both software developers and information technology while automating the process of software delivery </div><div> </div><div>BBK Worldwide offers a non-hierarchical corporate structure, one that promotes teamwork and creativity throughout the company. BBK Worldwide offers a unique environment not only in which to work, but also to create and contribute. This position is an excellent opportunity for the developer with the right skill set who wants to contribute at all levels to enhancing and advancing the company’s proprietary, web-based software</div><div> </div><div>Requirements for this position:</div><div>·       Degree in computer science or related field, advanced degree preferred</div><div>·       Minimum of 5 years of experience; 3 years for those with advanced degrees</div><div>·       Fluency in written and spoken English language</div><div>·       Eligible and able to work full-time from our corporate headquarters in Needham, Massachusetts </div><div> </div><div>To apply for this position, please submit your resume and cover letter to <a href="mailto:jobs@bbkworldwide.com"><strong>jobs@bbkworldwide.com</strong></a>. </div><div> </div><div>To find out more about us, please visit <a href="http://www.bbkworldwide.com"><strong>www.bbkworldwide.com</strong></a>. </div> + +<p><strong>To apply:</strong> <a href="https://weworkremotely.com/remote-jobs/bbk-worldwide-software-developer">https://weworkremotely.com/remote-jobs/bbk-worldwide-software-developer</a></p> + + Wed, 28 Oct 2020 17:41:17 +0000 + https://weworkremotely.com/remote-jobs/bbk-worldwide-software-developer + https://weworkremotely.com/remote-jobs/bbk-worldwide-software-developer + + + + Boatyard: Senior Ruby on Rails Engineer + Anywhere (100% Remote) Only + Programming + <img src="https://we-work-remotely.imgix.net/logos/0017/2684/logo.gif?ixlib=rails-4.0.0&w=50&h=50&dpr=2&fit=fill&auto=compress" /> + +<p> + <strong>Headquarters:</strong> Ft Lauderdale + <br /><strong>URL:</strong> <a href="https://www.boatyard.com/">https://www.boatyard.com/</a> +</p> + +<div>Boatyard is an exciting tech company on a mission to connect the world of boating through our award-winning mobile app and SaaS software.</div><div><br></div><div>We are looking for a senior engineer who would embrace the opportunity to take a leadership role with our fast growing startup. Our ideal candidate has 7+ years experience as a Ruby on Rails engineer, with 3+ years in a leadership role. This is a unique opportunity for a self-driven, collaborative, problem solver who is passionate about delivering quality code.  If this sounds like you, we'd love to hear from you.<br><br>Candidates must be located within two hours of EST. Seeking independent engineers only. No agencies, dev shops or recruiters please. </div><div><br></div><div>Our tech stack :</div><div><br></div><ul> +<li>Ruby on Rails</li> +<li>Postgres Database (on AWS RDS)</li> +<li>Apache Web Server</li> +<li>Amazon Web Services for Hosting (EC2 instances)</li> +<li>WePay Payments (payment integration)</li> +<li>JSON based Boatyard API</li> +<li>GraphQL</li> +<li>Apollo</li> +</ul><div><br></div><div>Front end:</div><ul> +<li>React Native</li> +<li>React</li> +</ul><div> +<br>Responsibilities<br><br><br><br> +</div><ul> +<li>Collaborating with the operations team on sprint planning, scheduling and implementation</li> +<li>Creating and maintaining a feature roadmap with founder and key stakeholders</li> +<li>Managing the development team to ensure on-time delivery of product features and completion of sprints</li> +<li>Implement unit testing and plan for ongoing development of a test suite Managing app development projects</li> +<li>Perform additional development projects, such as email campaigns, website development, etc.</li> +<li>Manage admin/developer accounts for various third-party applications</li> +<li>Performing code reviews and QA</li> +<li>Improving existing APIs and building new, unit-tested APIs, Creating unit tests</li> +<li>Ensure that features are being delivered efficiently and on-time and as the most practical solution to meet business goals.</li> +</ul><div><br></div><div>Requirements</div><div><br></div><ul> +<li>5+ years experience developing Ruby on Rails apps that interface with Restful APIs </li> +<li>Experience with Third-Party API Integrations</li> +<li>Possess excellent communication skills</li> +<li>Fluent speaking and writing in English</li> +<li>Write high quality, well-structured code</li> +<li>Available to work 40+ hours/week during EST work hours</li> +<li>Experience with graphQL a plus</li> +<li>Experience with DevOps (AWS) a plus</li> +</ul><div><br></div><div>Why Boatyard?</div><div><br></div><ul> +<li>You would be joining a team that believes our work is a craft, and takes great pride in what we do. </li> +<li>We are an innovation lab within a larger company, combining the culture of a startup with the security of working at a corporation.</li> +<li>Great team culture</li> +<li>Lots of opportunity for growth</li> +</ul><div><br></div> + +<p><strong>To apply:</strong> <a href="https://weworkremotely.com/remote-jobs/boatyard-senior-ruby-on-rails-engineer">https://weworkremotely.com/remote-jobs/boatyard-senior-ruby-on-rails-engineer</a></p> + + Wed, 28 Oct 2020 17:31:17 +0000 + https://weworkremotely.com/remote-jobs/boatyard-senior-ruby-on-rails-engineer + https://weworkremotely.com/remote-jobs/boatyard-senior-ruby-on-rails-engineer + + + diff --git a/tests/jobs_weworkremotely/test_spider.py b/tests/jobs_weworkremotely/test_spider.py new file mode 100644 index 0000000..966ea8c --- /dev/null +++ b/tests/jobs_weworkremotely/test_spider.py @@ -0,0 +1,103 @@ +from datetime import date +from pathlib import Path + +import pytest +from scrapy.http import HtmlResponse, XmlResponse + +from juniorguru_plucker.jobs_weworkremotely.spider import Spider, parse_date + + +FIXTURES_DIR = Path(__file__).parent + + +def test_spider_parse(): + response = XmlResponse( + "https://example.com/example/", + body=Path(FIXTURES_DIR / "remote-programming-jobs.rss").read_bytes(), + ) + requests = list(Spider().parse(response)) + + assert len(requests) == 4 + + data = requests[0].cb_kwargs["feed_data"] + assert data["title"] == "10up: Senior UI Engineer" + assert data["remote"] is True + assert data["first_seen_on"] == date(2020, 10, 28) + assert data["company_logo_urls"] == [] + assert "
  • Moderate PHP experience.
    • " in data["description_html"] + + data = requests[1].cb_kwargs["feed_data"] + assert data["company_logo_urls"] == [ + "https://wwr-pro.s3.amazonaws.com/logos/0017/2713/logo.gif" + ] + + +def test_spider_parse_job(): + feed_data = dict(remote=True) + response = HtmlResponse( + "https://example.com/example/", + body=Path(FIXTURES_DIR / "job.html").read_bytes(), + ) + jobs = list(Spider().parse_job(response, feed_data)) + + assert len(jobs) == 1 + + job = jobs[0] + + assert sorted(job.keys()) == sorted( + [ + "title", + "url", + "company_name", + "company_url", + "employment_types", + "first_seen_on", + "description_html", + "company_logo_urls", + "remote", + "locations_raw", + "source", + "source_urls", + ] + ) + assert job["title"] == "DevOps Engineer, Kubernetes, AWS/GCP" + assert job["url"] == "https://example.com/example/" + assert job["company_name"] == "Bluelight Consulting" + assert job["company_url"] == "https://bluelight.co" + assert job["remote"] is True + assert job["first_seen_on"] == date(2020, 10, 20) + assert job["company_logo_urls"] == [ + "https://we-work-remotely.imgix.net/logos/0017/2301/logo.gif?ixlib=rails-4.0.0&w=50&h=50&dpr=2&fit=fill&auto=compress" + ] + assert job["source"] == "jobs-weworkremotely" + assert job["source_urls"] == ["https://example.com/example/"] + assert "
  • Kubernetes Certificates
    • " in job["description_html"] + + +def test_spider_parse_job_no_image(): + response = HtmlResponse( + "https://example.com/example/", + body=Path(FIXTURES_DIR / "job_no_image.html").read_bytes(), + ) + job = next(Spider().parse_job(response, {})) + + assert job.get("company_logo_urls") is None + + +def test_spider_parse_job_json_decode_error_gets_skipped(): + response = HtmlResponse( + "https://example.com/example/", + body=Path(FIXTURES_DIR / "job_json_decode_error.html").read_bytes(), + ) + jobs = Spider().parse_job(response, {}) + + with pytest.raises(StopIteration): + next(jobs) + + +def test_parse_date(): + assert parse_date("2024-01-02 13:53:45 UTC") == date(2024, 1, 2) + + +def test_parse_date_none(): + assert parse_date(None) is None