git clone https://github.com/eghuro/crawlcheck crawlcheck

cd crawlcheck
pip install -r requirements.txt

---
version: 1.6                    # configuration format version
database: crawlcheck.sqlite     # sqlite database file
maxDepth: 10                    # max amount of links followed from any entry point (default: 0 meaning unlimited)
agent: "Crawlcheck/1.1"         # user agent used (default: Crawlcheck/1.1)
logfile: cc.log                 # where to store logs
maxContentLength: 2000000       # max file size to download
pluginDir: plugin               # where to look for plugins (including subfolders, default: 'plugin')
timeout: 1                      # timeout for networking (default: 1)
cleandb: True                   # clean database before execution
initdb: True                    # initialize database
report: "http://localhost:5000" # report REST API
cleanreport: True               # clean entries in report before sending current
maxVolume: 100000000            # max 100 MB of tmp files (default: sys.maxsize)
maxAttempts: 2                  # attempts to download a web page (default: 3)
dbCacheLimit: 1000000           # cache up to 1M of DB queries
tmpPrefix: "Crawlcheck"         # prefix for temporary file names with downloaded content (default: Crawlcheck)
tmpSuffix: "content"            # suffix for temporary file names with downloaded content (default: content)
tmpDir: "/tmp/"                 # where to store temporary files (default: /tmp/)
dbCacheLimit: 100000            # amount of cached database queries (default: sys.maxsize)
urlLimit: 10000000              # limit on seen URIs
verifyHttps: True               # verify HTTPS? (default: False)
cores: 2                        # amount of cores available (eg. for paralel report payload generation)
recordParams: False             # record request data or URL params? (default: True)
recordHeaders: False            # record response headers? (default: True)
sitemap-file: "sitemap.xml"     # where to store generated sitemap.xml
sitemap-regex: "https?://ksp.mff.cuni.cz(/.*)?" # regex for sitemap generator
yaml-out-file: "cc.yml"         # where to write YAML report
report-file: "report"           # where to write PDF report (.pdf will be added automatically)


# other parameters used by plugins written as ```key: value```

urls:
 -
  url: "http://mj.ucw.cz/vyuka/.+"
  plugins: # which plugins are allowed for given URL
       - linksFinder
       - tidyHtmlValidator
       - tinycss
       - css_scraper
       - formChecker
       - seoimg
       - seometa
       - dupdeteict
       - non_semantic_html
 -
  url: "http://mj.ucw.cz/" #test links (HEAD request) only
  plugins:

filters:  #Filters (plugins of category header and filter) that can be used
 - depth
 - robots
 - contentLength
 - canonical
 - acceptedType
 - acceptedUri
 - uri_normalizer
 - expectedType

# filters: True
# alternative option to allow all available filters
# can be passed on command line using --param

postprocess:
 - sitemap_generator
 - report_exporter
 - yaml_exporter

# postprocess: True
# alternative option to allow all available postprocessors
# can be passed on command line using --param

entryPoints: # where to start
# Note, that once URI get's to the database it's no longer being requested 
# (beware of repeated starts, if entry point remains in the database execution won't 
# start from this entry point)
 - "http://mj.ucw.cz/vyuka/"

$ cd [root]/crawlcheck/src/
$ python checker/ [config.yml]

[Core]
Name = Human readable plugin name
Module = my_new_plugin

[Documentation]
Author = Your Name
Version = 0.0
Description = My New Plugin

from yapsy.IPlugin import IPlugin
from common import PluginType
from filter import FilterException  # for headers and filters


class MyPlugin(IPlugin):

    category = PluginType.CHECKER # pick appropriate type
    id = myPlugin
    contentTypes = ["text/html"] # accepted content types (checkers & crawlers)

    def acceptType(ctype): # alternatively a method resolving more complex content-type rules
        return True

    def setJournal(self, journal):
        # record journal somewhere - all categories

    def setQueue(self, queue):
        # record queue somewhere - if needed

    def setConf(self, conf):
        # record configuration - only headers and filters

    def check(self, transaction):
        # implement the checking logic here for crawlers and checkers

    def filter(self, transaction):
        # implement the filtering logic here for filters and headers
        # raise FilterException to filter the transaction out

    def setDb(self, db):
        # record DB somewhere - only postprocessors

    def process(self):
        # implement the postprocessing logic here for postprocessor