From fc828dbe389b5cb43cac51a1feedc45c0daa129b Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sun, 24 Nov 2013 15:06:23 +0100 Subject: [PATCH] Bunch of fixes --- README.md | 7 +++++++ pythonwhois/parse.py | 10 ++++------ test/target_default/nominet.org.uk | 2 +- test/target_default/whirlpool.net.au | 2 +- test/target_normalized/nominet.org.uk | 2 +- test/target_normalized/whirlpool.net.au | 2 +- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 65a40e9..35f2d8f 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ If any of those apply, don't hesitate to file an issue! The goal is 100% coverag Feel free to fork and submit pull requests! If you change any parsing or normalization logic, ensure to run the full test suite before opening a pull request. Instructions for that are below. +All commands are relative to the root directory of the repository. + **Pull requests that do _not_ include output from test.py will be rejected!** ### Adding new WHOIS data to the testing set @@ -71,3 +73,8 @@ Make sure to verify (using `pwhois` or otherwise) that the WHOIS data for the do ./test.py run thedomain.com +### Generating documentation + +You need [ZippyDoc](http://cryto.net/zippydoc) (which can be installed through `pip install zippydoc`). + + zpy2html doc/*.zpy diff --git a/pythonwhois/parse.py b/pythonwhois/parse.py index daca3cf..953e697 100644 --- a/pythonwhois/parse.py +++ b/pythonwhois/parse.py @@ -218,12 +218,10 @@ def parse_raw_whois(raw_data, normalized=[]): data['emails'] = [email for email in data["emails"] if email not in known_emails] except KeyError, e: pass # Not present - - try: - if len(data['emails']) == 0: - del data['emails'] - except KeyError, e: - pass + + for key in data.keys(): + if data[key] is None or len(data[key]) == 0: + del data[key] data["raw"] = raw_data diff --git a/test/target_default/nominet.org.uk b/test/target_default/nominet.org.uk index db5239e..959ac18 100644 --- a/test/target_default/nominet.org.uk +++ b/test/target_default/nominet.org.uk @@ -1 +1 @@ -{"updated_date": ["2013-02-06T00:00:00"], "contacts": {"admin": null, "tech": null, "registrant": {"city": "Oxford", "name": "Nominet UK", "state": "Oxon", "street": "Minerva House\nEdmund Halley Road\nOxford Science Park", "country": "United Kingdom", "postalcode": "OX4 4DQ"}, "billing": null}, "nameservers": ["nom-ns1.nominet.org.uk", "nom-ns2.nominet.org.uk", "nom-ns3.nominet.org.uk"], "creation_date": null, "raw": ["\n Domain name:\n nominet.org.uk\n\n Registrant:\n Nominet UK\n\n Registrant type:\n UK Limited Company, (Company number: 3203859)\n\n Registrant's address:\n Minerva House\n Edmund Halley Road\n Oxford Science Park\n Oxford\n Oxon\n OX4 4DQ\n United Kingdom\n\n Registrar:\n No registrar listed. This domain is registered directly with Nominet.\n\n Relevant dates:\n Registered on: before Aug-1996\n Last updated: 06-Feb-2013\n\n Registration status:\n No registration status listed.\n\n Name servers:\n nom-ns1.nominet.org.uk 213.248.199.16\n nom-ns2.nominet.org.uk 195.66.240.250 2a01:40:1001:37::2\n nom-ns3.nominet.org.uk 213.219.13.194\n\n DNSSEC:\n Signed\n\n WHOIS lookup made at 14:56:34 23-Nov-2013\n\n-- \nThis WHOIS information is provided for free by Nominet UK the central registry\nfor .uk domain names. This information and the .uk WHOIS are:\n\n Copyright Nominet UK 1996 - 2013.\n\nYou may not access the .uk WHOIS or use any data from it except as permitted\nby the terms of use available in full at http://www.nominet.org.uk/whoisterms, which\nincludes restrictions on: (A) use of the data for advertising, or its\nrepackaging, recompilation, redistribution or reuse (B) obscuring, removing\nor hiding any or all of this notice and (C) exceeding query rate or volume\nlimits. The data is provided on an 'as-is' basis and may lag behind the\nregister. Access may be withdrawn or restricted at any time. \n\n"], "registrar": ["No registrar listed. This domain is registered directly with Nominet."]} \ No newline at end of file +{"updated_date": ["2013-02-06T00:00:00"], "nameservers": ["nom-ns1.nominet.org.uk", "nom-ns2.nominet.org.uk", "nom-ns3.nominet.org.uk"], "contacts": {"admin": null, "tech": null, "registrant": {"city": "Oxford", "name": "Nominet UK", "state": "Oxon", "street": "Minerva House\nEdmund Halley Road\nOxford Science Park", "country": "United Kingdom", "postalcode": "OX4 4DQ"}, "billing": null}, "registrar": ["No registrar listed. This domain is registered directly with Nominet."], "raw": ["\n Domain name:\n nominet.org.uk\n\n Registrant:\n Nominet UK\n\n Registrant type:\n UK Limited Company, (Company number: 3203859)\n\n Registrant's address:\n Minerva House\n Edmund Halley Road\n Oxford Science Park\n Oxford\n Oxon\n OX4 4DQ\n United Kingdom\n\n Registrar:\n No registrar listed. This domain is registered directly with Nominet.\n\n Relevant dates:\n Registered on: before Aug-1996\n Last updated: 06-Feb-2013\n\n Registration status:\n No registration status listed.\n\n Name servers:\n nom-ns1.nominet.org.uk 213.248.199.16\n nom-ns2.nominet.org.uk 195.66.240.250 2a01:40:1001:37::2\n nom-ns3.nominet.org.uk 213.219.13.194\n\n DNSSEC:\n Signed\n\n WHOIS lookup made at 14:56:34 23-Nov-2013\n\n-- \nThis WHOIS information is provided for free by Nominet UK the central registry\nfor .uk domain names. This information and the .uk WHOIS are:\n\n Copyright Nominet UK 1996 - 2013.\n\nYou may not access the .uk WHOIS or use any data from it except as permitted\nby the terms of use available in full at http://www.nominet.org.uk/whoisterms, which\nincludes restrictions on: (A) use of the data for advertising, or its\nrepackaging, recompilation, redistribution or reuse (B) obscuring, removing\nor hiding any or all of this notice and (C) exceeding query rate or volume\nlimits. The data is provided on an 'as-is' basis and may lag behind the\nregister. Access may be withdrawn or restricted at any time. \n\n"]} \ No newline at end of file diff --git a/test/target_default/whirlpool.net.au b/test/target_default/whirlpool.net.au index 7d6f2a4..d2eb820 100644 --- a/test/target_default/whirlpool.net.au +++ b/test/target_default/whirlpool.net.au @@ -1 +1 @@ -{"status": ["ok"], "updated_date": ["2012-02-06T09:28:40"], "contacts": {"admin": null, "tech": {"handle": "WRSI1010", "name": "Simon Wright"}, "registrant": {"organization": "Whirlpool Broadband Multimedia", "handle": "WRSI1010", "name": "Simon Wright"}, "billing": null}, "nameservers": ["ns0.bulletproof.net.au", "ns1.bulletproof.net.au", "ns1.bulletproofnetworks.net", "ns0.bulletproofnetworks.net"], "creation_date": null, "raw": ["Domain Name: whirlpool.net.au\nLast Modified: 06-Feb-2012 09:28:40 UTC\nRegistrar ID: NetRegistry\nRegistrar Name: NetRegistry\nStatus: ok\n\nRegistrant: Simon Wright\nEligibility Type: Registered Business\nEligibility Name: Whirlpool Broadband Multimedia\nEligibility ID: NSW BN BN98319722\n\nRegistrant Contact ID: WRSI1010\nRegistrant Contact Name: Simon Wright\nRegistrant Contact Email: Visit whois.ausregistry.com.au for Web based WhoIs\n\nTech Contact ID: WRSI1010\nTech Contact Name: Simon Wright\nTech Contact Email: Visit whois.ausregistry.com.au for Web based WhoIs\n\nName Server: ns0.bulletproof.net.au\nName Server IP: 202.44.98.24\nName Server: ns1.bulletproof.net.au\nName Server IP: 64.71.152.56\nName Server: ns1.bulletproofnetworks.net\nName Server: ns0.bulletproofnetworks.net\n\n"], "registrar": ["NetRegistry"]} \ No newline at end of file +{"status": ["ok"], "updated_date": ["2012-02-06T09:28:40"], "contacts": {"admin": null, "tech": {"handle": "WRSI1010", "name": "Simon Wright"}, "registrant": {"organization": "Whirlpool Broadband Multimedia", "handle": "WRSI1010", "name": "Simon Wright"}, "billing": null}, "nameservers": ["ns0.bulletproof.net.au", "ns1.bulletproof.net.au", "ns1.bulletproofnetworks.net", "ns0.bulletproofnetworks.net"], "raw": ["Domain Name: whirlpool.net.au\nLast Modified: 06-Feb-2012 09:28:40 UTC\nRegistrar ID: NetRegistry\nRegistrar Name: NetRegistry\nStatus: ok\n\nRegistrant: Simon Wright\nEligibility Type: Registered Business\nEligibility Name: Whirlpool Broadband Multimedia\nEligibility ID: NSW BN BN98319722\n\nRegistrant Contact ID: WRSI1010\nRegistrant Contact Name: Simon Wright\nRegistrant Contact Email: Visit whois.ausregistry.com.au for Web based WhoIs\n\nTech Contact ID: WRSI1010\nTech Contact Name: Simon Wright\nTech Contact Email: Visit whois.ausregistry.com.au for Web based WhoIs\n\nName Server: ns0.bulletproof.net.au\nName Server IP: 202.44.98.24\nName Server: ns1.bulletproof.net.au\nName Server IP: 64.71.152.56\nName Server: ns1.bulletproofnetworks.net\nName Server: ns0.bulletproofnetworks.net\n\n"], "registrar": ["NetRegistry"]} \ No newline at end of file diff --git a/test/target_normalized/nominet.org.uk b/test/target_normalized/nominet.org.uk index db5239e..959ac18 100644 --- a/test/target_normalized/nominet.org.uk +++ b/test/target_normalized/nominet.org.uk @@ -1 +1 @@ -{"updated_date": ["2013-02-06T00:00:00"], "contacts": {"admin": null, "tech": null, "registrant": {"city": "Oxford", "name": "Nominet UK", "state": "Oxon", "street": "Minerva House\nEdmund Halley Road\nOxford Science Park", "country": "United Kingdom", "postalcode": "OX4 4DQ"}, "billing": null}, "nameservers": ["nom-ns1.nominet.org.uk", "nom-ns2.nominet.org.uk", "nom-ns3.nominet.org.uk"], "creation_date": null, "raw": ["\n Domain name:\n nominet.org.uk\n\n Registrant:\n Nominet UK\n\n Registrant type:\n UK Limited Company, (Company number: 3203859)\n\n Registrant's address:\n Minerva House\n Edmund Halley Road\n Oxford Science Park\n Oxford\n Oxon\n OX4 4DQ\n United Kingdom\n\n Registrar:\n No registrar listed. This domain is registered directly with Nominet.\n\n Relevant dates:\n Registered on: before Aug-1996\n Last updated: 06-Feb-2013\n\n Registration status:\n No registration status listed.\n\n Name servers:\n nom-ns1.nominet.org.uk 213.248.199.16\n nom-ns2.nominet.org.uk 195.66.240.250 2a01:40:1001:37::2\n nom-ns3.nominet.org.uk 213.219.13.194\n\n DNSSEC:\n Signed\n\n WHOIS lookup made at 14:56:34 23-Nov-2013\n\n-- \nThis WHOIS information is provided for free by Nominet UK the central registry\nfor .uk domain names. This information and the .uk WHOIS are:\n\n Copyright Nominet UK 1996 - 2013.\n\nYou may not access the .uk WHOIS or use any data from it except as permitted\nby the terms of use available in full at http://www.nominet.org.uk/whoisterms, which\nincludes restrictions on: (A) use of the data for advertising, or its\nrepackaging, recompilation, redistribution or reuse (B) obscuring, removing\nor hiding any or all of this notice and (C) exceeding query rate or volume\nlimits. The data is provided on an 'as-is' basis and may lag behind the\nregister. Access may be withdrawn or restricted at any time. \n\n"], "registrar": ["No registrar listed. This domain is registered directly with Nominet."]} \ No newline at end of file +{"updated_date": ["2013-02-06T00:00:00"], "nameservers": ["nom-ns1.nominet.org.uk", "nom-ns2.nominet.org.uk", "nom-ns3.nominet.org.uk"], "contacts": {"admin": null, "tech": null, "registrant": {"city": "Oxford", "name": "Nominet UK", "state": "Oxon", "street": "Minerva House\nEdmund Halley Road\nOxford Science Park", "country": "United Kingdom", "postalcode": "OX4 4DQ"}, "billing": null}, "registrar": ["No registrar listed. This domain is registered directly with Nominet."], "raw": ["\n Domain name:\n nominet.org.uk\n\n Registrant:\n Nominet UK\n\n Registrant type:\n UK Limited Company, (Company number: 3203859)\n\n Registrant's address:\n Minerva House\n Edmund Halley Road\n Oxford Science Park\n Oxford\n Oxon\n OX4 4DQ\n United Kingdom\n\n Registrar:\n No registrar listed. This domain is registered directly with Nominet.\n\n Relevant dates:\n Registered on: before Aug-1996\n Last updated: 06-Feb-2013\n\n Registration status:\n No registration status listed.\n\n Name servers:\n nom-ns1.nominet.org.uk 213.248.199.16\n nom-ns2.nominet.org.uk 195.66.240.250 2a01:40:1001:37::2\n nom-ns3.nominet.org.uk 213.219.13.194\n\n DNSSEC:\n Signed\n\n WHOIS lookup made at 14:56:34 23-Nov-2013\n\n-- \nThis WHOIS information is provided for free by Nominet UK the central registry\nfor .uk domain names. This information and the .uk WHOIS are:\n\n Copyright Nominet UK 1996 - 2013.\n\nYou may not access the .uk WHOIS or use any data from it except as permitted\nby the terms of use available in full at http://www.nominet.org.uk/whoisterms, which\nincludes restrictions on: (A) use of the data for advertising, or its\nrepackaging, recompilation, redistribution or reuse (B) obscuring, removing\nor hiding any or all of this notice and (C) exceeding query rate or volume\nlimits. The data is provided on an 'as-is' basis and may lag behind the\nregister. Access may be withdrawn or restricted at any time. \n\n"]} \ No newline at end of file diff --git a/test/target_normalized/whirlpool.net.au b/test/target_normalized/whirlpool.net.au index 7d6f2a4..d2eb820 100644 --- a/test/target_normalized/whirlpool.net.au +++ b/test/target_normalized/whirlpool.net.au @@ -1 +1 @@ -{"status": ["ok"], "updated_date": ["2012-02-06T09:28:40"], "contacts": {"admin": null, "tech": {"handle": "WRSI1010", "name": "Simon Wright"}, "registrant": {"organization": "Whirlpool Broadband Multimedia", "handle": "WRSI1010", "name": "Simon Wright"}, "billing": null}, "nameservers": ["ns0.bulletproof.net.au", "ns1.bulletproof.net.au", "ns1.bulletproofnetworks.net", "ns0.bulletproofnetworks.net"], "creation_date": null, "raw": ["Domain Name: whirlpool.net.au\nLast Modified: 06-Feb-2012 09:28:40 UTC\nRegistrar ID: NetRegistry\nRegistrar Name: NetRegistry\nStatus: ok\n\nRegistrant: Simon Wright\nEligibility Type: Registered Business\nEligibility Name: Whirlpool Broadband Multimedia\nEligibility ID: NSW BN BN98319722\n\nRegistrant Contact ID: WRSI1010\nRegistrant Contact Name: Simon Wright\nRegistrant Contact Email: Visit whois.ausregistry.com.au for Web based WhoIs\n\nTech Contact ID: WRSI1010\nTech Contact Name: Simon Wright\nTech Contact Email: Visit whois.ausregistry.com.au for Web based WhoIs\n\nName Server: ns0.bulletproof.net.au\nName Server IP: 202.44.98.24\nName Server: ns1.bulletproof.net.au\nName Server IP: 64.71.152.56\nName Server: ns1.bulletproofnetworks.net\nName Server: ns0.bulletproofnetworks.net\n\n"], "registrar": ["NetRegistry"]} \ No newline at end of file +{"status": ["ok"], "updated_date": ["2012-02-06T09:28:40"], "contacts": {"admin": null, "tech": {"handle": "WRSI1010", "name": "Simon Wright"}, "registrant": {"organization": "Whirlpool Broadband Multimedia", "handle": "WRSI1010", "name": "Simon Wright"}, "billing": null}, "nameservers": ["ns0.bulletproof.net.au", "ns1.bulletproof.net.au", "ns1.bulletproofnetworks.net", "ns0.bulletproofnetworks.net"], "raw": ["Domain Name: whirlpool.net.au\nLast Modified: 06-Feb-2012 09:28:40 UTC\nRegistrar ID: NetRegistry\nRegistrar Name: NetRegistry\nStatus: ok\n\nRegistrant: Simon Wright\nEligibility Type: Registered Business\nEligibility Name: Whirlpool Broadband Multimedia\nEligibility ID: NSW BN BN98319722\n\nRegistrant Contact ID: WRSI1010\nRegistrant Contact Name: Simon Wright\nRegistrant Contact Email: Visit whois.ausregistry.com.au for Web based WhoIs\n\nTech Contact ID: WRSI1010\nTech Contact Name: Simon Wright\nTech Contact Email: Visit whois.ausregistry.com.au for Web based WhoIs\n\nName Server: ns0.bulletproof.net.au\nName Server IP: 202.44.98.24\nName Server: ns1.bulletproof.net.au\nName Server IP: 64.71.152.56\nName Server: ns1.bulletproofnetworks.net\nName Server: ns0.bulletproofnetworks.net\n\n"], "registrar": ["NetRegistry"]} \ No newline at end of file