spider-lib 1.3.0

A Rust-based web scraping framework inspired by Scrapy (Python).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

.PHONY: publish-all sync-versions check-all-features build-all-features clean check-fast

# Build/check with all feature combinations (auto-discovered from Cargo.toml)
check-all-features:
	@chmod +x ./scripts/build_all_features.sh
	@./scripts/build_all_features.sh

build-all-features:
	@echo "Building with all features (release mode)..."
	@cargo build --release --all-features

# Fast check for development (no optimizations)
check-fast:
	@echo "Running fast check (dev, no optimizations)..."
	@CARGO_INCREMENTAL=1 cargo check --workspace --all-targets

# Check specific package only
# Usage: make check-pkg PKG=spider-util
check-pkg:
	@echo "Checking $(PKG)..."
	@cargo check -p $(PKG)

# Clean build artifacts
clean:
	@echo "Cleaning build artifacts..."
	@cargo clean

# Check with timing info
check-timing:
	@echo "Running check with timing information..."
	@CARGO_INCREMENTAL=1 cargo check --workspace --all-targets --timings

sync-versions:
	@chmod +x ./scripts/sync_versions.sh
	@./scripts/sync_versions.sh

publish-submodule:
	@echo "Checking current version against crates.io for submodule $(SUBMODULE)..."
	@cd $(SUBMODULE) && \
	CURRENT_VERSION=$$(grep -m 1 "^version =" Cargo.toml | cut -d '"' -f 2); \
	CRATE_NAME=$$(grep -m 1 "^name =" Cargo.toml | cut -d '"' -f 2); \
	echo "Current crate: $$CRATE_NAME, version: $$CURRENT_VERSION"; \
	if cargo search "$$CRATE_NAME" --limit 1 | grep -q "$$CRATE_NAME = \"$$CURRENT_VERSION\""; then \
		echo "Version $$CURRENT_VERSION already exists on crates.io, skipping publish for $$CRATE_NAME."; \
	else \
		echo "Local version ($$CURRENT_VERSION) not found on crates.io, publishing $$CRATE_NAME..."; \
		cargo publish --all-features --allow-dirty; \
	fi

publish-main:
	@echo "Checking current version against crates.io for main repository..."
	@CURRENT_VERSION=$$(grep -m 1 "^version =" Cargo.toml | cut -d '"' -f 2); \
	CRATE_NAME=$$(grep -m 1 "^name =" Cargo.toml | cut -d '"' -f 2); \
	echo "Current crate: $$CRATE_NAME, version: $$CURRENT_VERSION"; \
	if cargo search "$$CRATE_NAME" --limit 1 | grep -q "$$CRATE_NAME = \"$$CURRENT_VERSION\""; then \
		echo "Version $$CURRENT_VERSION already exists on crates.io, skipping publish for $$CRATE_NAME."; \
	else \
		echo "Local version ($$CURRENT_VERSION) not found on crates.io, publishing $$CRATE_NAME..."; \
		cargo publish --all-features --allow-dirty; \
	fi

publish-all:
	@$(MAKE) sync-versions
	@echo "Publishing all packages with spider-lib (main repo) last..."
	@$(MAKE) SUBMODULE="spider-util" publish-submodule
	@$(MAKE) SUBMODULE="spider-macro" publish-submodule
	@$(MAKE) SUBMODULE="spider-middleware" publish-submodule
	@$(MAKE) SUBMODULE="spider-pipeline" publish-submodule
	@$(MAKE) SUBMODULE="spider-downloader" publish-submodule
	@$(MAKE) SUBMODULE="spider-core" publish-submodule
	@$(MAKE) publish-main
	@echo "Publishing complete!"