| | 1 | [[PageOutline]] |
| | 2 | = Swish-e = |
| | 3 | |
| | 4 | == '''swishbuild''' Building Swish-e package == |
| | 5 | |
| | 6 | {{{ |
| | 7 | swish- |
| | 8 | swishbuild- |
| | 9 | swishbuild-again |
| | 10 | }}} |
| | 11 | |
| | 12 | == '''Important == |
| | 13 | * Do not run swish-e as root. |
| | 14 | |
| | 15 | == '''swish-e'' configuration (spider.conf) == |
| | 16 | |
| | 17 | {{{ |
| | 18 | my %dayabay = ( |
| | 19 | email => 'tianxc@ihep.ac.cn', |
| | 20 | base_url => 'http://dayabay.ihep.ac.cn/', |
| | 21 | delay_sec => '1', |
| | 22 | max_depth => '1', |
| | 23 | credentials => 'dayabay:3quarks' |
| | 24 | ); |
| | 25 | |
| | 26 | my %documents = ( |
| | 27 | email => 'tianxc@ihep.ac.cn', |
| | 28 | base_url => 'http://dayabay.bnl.gov/private/documents/', |
| | 29 | delay_sec => '1', |
| | 30 | max_depth => '1', |
| | 31 | credentials => 'dayabay:3quarks' |
| | 32 | ); |
| | 33 | |
| | 34 | my %docdb = ( |
| | 35 | email => 'tianxc@ihep.ac.cn', |
| | 36 | base_url => 'http://dayabay.ihep.ac.cn/cgi-bin/DocDB/ListBy?alldocs=1', |
| | 37 | delay_sec => '1', |
| | 38 | max_depth => '1', |
| | 39 | credentials => 'dayabay:3quarks' |
| | 40 | ); |
| | 41 | |
| | 42 | my %engdb = ( |
| | 43 | email => 'tianxc@ihep.ac.cn', |
| | 44 | base_url => 'http://dayabay.ihep.ac.cn/cgi-bin/EngDB/ListBy?alldocs=1', |
| | 45 | delay_sec => '1', |
| | 46 | max_depth => '1', |
| | 47 | credentials => 'dayabay:3quarks' |
| | 48 | ); |
| | 49 | |
| | 50 | my %internal = ( |
| | 51 | email => 'tianxc@ihep.ac.cn', |
| | 52 | base_url => 'http://dayabay.ihep.ac.cn/internal/', |
| | 53 | delay_sec => '1', |
| | 54 | max_depth => '1', |
| | 55 | credentials => 'dayabay:3quarks' |
| | 56 | ); |
| | 57 | |
| | 58 | my %publicwiki = ( |
| | 59 | email => 'tianxc@ihep.ac.cn', |
| | 60 | base_url => 'https://wiki.bnl.gov/dayabay/index.php?title=Main_Page', |
| | 61 | delay_sec => '1', |
| | 62 | max_depth => '2', |
| | 63 | ); |
| | 64 | |
| | 65 | my %privatewiki = ( |
| | 66 | email => 'tianxc@ihep.ac.cn', |
| | 67 | base_url => 'https://wiki.bnl.gov/dayabay-private/index.php?title=Main_Page', |
| | 68 | delay_sec => '1', |
| | 69 | max_depth => '2', |
| | 70 | credentials => 'dayabay:3quarks' |
| | 71 | ); |
| | 72 | |
| | 73 | my %repository = ( |
| | 74 | email => 'tianxc@ihep.ac.cn', |
| | 75 | base_url => 'http://dayabay.ihep.ac.cn/tracs/dybsvn/browser/', |
| | 76 | delay_sec => '1', |
| | 77 | max_depth => '10', |
| | 78 | credentials => 'dayabay:3quarks' |
| | 79 | ); |
| | 80 | |
| | 81 | my %trac = ( |
| | 82 | email => 'tianxc@ihep.ac.cn', |
| | 83 | base_url => 'http://dayabay.ihep.ac.cn/tracs/dybsvn', |
| | 84 | delay_sec => '1', |
| | 85 | max_depth => '2', |
| | 86 | credentials => 'dayabay:3quarks' |
| | 87 | ); |
| | 88 | |
| | 89 | @servers = ( \%dayabay, \%documents, \%docdb, \%internal, \%engdb, \%publicwiki, \%privatewiki, \%repository, \%trac ); |
| | 90 | 1; |
| | 91 | }}} |
| | 92 | |
| | 93 | == swish.conf == |
| | 94 | {{{ |
| | 95 | # Example configuration file |
| | 96 | |
| | 97 | # Tell Swish-e what to index (same as -i switch above): |
| | 98 | IndexDir spider.pl |
| | 99 | |
| | 100 | |
| | 101 | # And pass the name of spider config file to the spider: |
| | 102 | SwishProgParameters spider.conf |
| | 103 | |
| | 104 | |
| | 105 | # Tell Swish-e that .txt files are to use the text parser: |
| | 106 | IndexContents TXT* .txt |
| | 107 | |
| | 108 | |
| | 109 | # Otherwise, use the HTML parser: |
| | 110 | DefaultContents HTML* |
| | 111 | |
| | 112 | |
| | 113 | # Ask libxml2 to report any parsing errors and warnings or |
| | 114 | # any UTF-8 to 8859-1 conversion errors: |
| | 115 | ParserWarnLevel 9 |
| | 116 | }}} |
| | 117 | |
| | 118 | == Indexing == |
| | 119 | {{{ |
| | 120 | touch indexing_time.file |
| | 121 | /usr/local/bin/swish-e -c swish.conf -S prog |
| | 122 | }}} |
