-
Notifications
You must be signed in to change notification settings - Fork 1
/
example.yml
99 lines (99 loc) · 2.28 KB
/
example.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
env: example
bot_name: crawler
mongo_enable: false
mongo:
example:
uri: mongodb://127.0.0.1:27017
database: crawler
mysql_enable: false
mysql:
example:
uri: root@tcp(127.0.0.1:3306)
database: crawler
redis_enable: false
redis:
example:
addr: 127.0.0.1:6379
password:
db: 0
sqlite:
- name: sqlite
path: crawler.db
store:
# - name: minio
# type: # s3/oss/cos/minio/file
# endpoint: http://127.0.0.1:9000 # http://127.0.0.1:9000 file://tmp/
# region:
# id:
# key:
# bucket: crawler
- name: file
type: file # s3/oss/cos/minio/file
endpoint: # http://127.0.0.1:9000 file://tmp/
kafka_enable: false
kafka:
example:
uri: localhost:9092
log:
filename: /tmp/spider_{name}.log
long_file: false # true/false
level: info # debug/info/warn/error
proxy:
example:
request:
concurrency: 1
interval: 1000
timeout: 30
ok_http_codes:
- 200
retry_max_times: 10
http_proto: 2.0
header:
accept_encoding: gzip, deflate, br
api:
enable: true
https: true
port: 8090
access_key: 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
mock_server:
enable: true
host: https://localhost:8081
client_auth: 0
close_reason:
queue_timeout: 10
enable_ja3: false
enable_priority_queue: true
enable_stats_middleware: true
enable_dump_middleware: true
scheduler: memory # memory/redis/kafka
filter: memory # memory/redis
enable_filter_middleware: true
enable_file_middleware: true
enable_image_middleware: true
enable_http_middleware: true
enable_retry_middleware: true
enable_referrer_middleware: true
referrer_policy: NoReferrerPolicy
enable_http_auth_middleware: false
enable_cookie_middleware: true
enable_url_middleware: true
url_length_limit:
enable_compress_middleware: true
enable_decode_middleware: true
enable_redirect_middleware: true
redirect_max_times: 10
enable_chrome_middleware: true
enable_device_middleware: false
enable_proxy_middleware: true
enable_robots_txt_middleware: false
enable_record_error_middleware: false
enable_dump_pipeline: true
enable_file_pipeline: true
enable_image_pipeline: true
enable_filter_pipeline: true
enable_none_pipeline: true
enable_csv_pipeline: false
enable_json_lines_pipeline: false
enable_mongo_pipeline: false
enable_mysql_pipeline: false
enable_kafka_pipeline: false