Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/ci-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,13 @@ jobs:
run: |
pip install .[dev]

- name: Check formatting
run: |
make check-format

- name: Check linting
run: |
make check-fix

- name: Run tests
run: pytest
22 changes: 14 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
run: build
@docker run \
--rm \
-ti \
scraperwiki-python
test:
@pytest

build:
@docker build -t scraperwiki-python .
fix:
@ruff check --fix

.PHONY: run build
check-fix:
@ruff check

format:
@ruff format

check-format:
@ruff format --check

.PHONY: test fix check-fix format check-format
8 changes: 4 additions & 4 deletions benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@
import scraperwiki
import os

rows = [{'id': i, 'test': i * 2, 's': "abc"} for i in range(1000)]
rows = [{"id": i, "test": i * 2, "s": "abc"} for i in range(1000)]

try:
os.remove('scraperwiki.sqlite')
os.remove("scraperwiki.sqlite")
except FileNotFoundError:
pass

scraperwiki.sql.save(['id'], rows)
scraperwiki.sql.save(["id"], rows)

for i, row in enumerate(rows):
scraperwiki.sql.save(['id'], row)
scraperwiki.sql.save(["id"], row)
4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,14 @@ dependencies = [

[project.optional-dependencies]
dev = [
"ruff",
"pytest",
"lxml",
]

[tool.ruff]
line-length = 88

[tool.setuptools.packages.find]
where = ["."] # Or wherever your 'scraperwiki' package folder is located
include = ["scraperwiki*"]
4 changes: 2 additions & 2 deletions save_speedtest.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#! /usr/bin/env python3
import scraperwiki

rows = [{'id': i, 'test': i * 2, 's': "xx"*i} for i in range(10)]
rows = [{"id": i, "test": i * 2, "s": "xx" * i} for i in range(10)]

for i, row in enumerate(rows):
scraperwiki.sql.save(['id'], row)
scraperwiki.sql.save(["id"], row)
18 changes: 16 additions & 2 deletions scraperwiki/__init__.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,33 @@
'''
"""
Local version of ScraperWiki Utils, documentation here:
https://scraperwiki.com/docs/python/python_help_documentation/
'''
"""

from .utils import scrape, pdftoxml, status, swimport
from . import utils
from . import sql

# Compatibility
sqlite = sql

__all__ = [
"scrape",
"pdftoxml",
"status",
"swimport",
"utils",
"sql",
]


class Error(Exception):
"""All ScraperWiki exceptions are instances of this class
(usually via a subclass)."""

pass


class CPUTimeExceededError(Error):
"""CPU time limit exceeded."""

pass
Loading
Loading