#!/usr/bin/perl # Project: Web Reference Database (refbase) # Copyright: Matthias Steffens and the file's # original author(s). # # This code is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY. Please see the GNU General Public # License for more details. # # File: ./contrib/command_line/refbase_import # Repository: $HeadURL: file:///svn/p/refbase/code/branches/bleeding-edge/contrib/command_line/refbase_import $ # Author(s): Matthias Steffens # # Created: 06-Jun-06, 18:00 # Modified: $Date: 2008-10-30 18:16:54 +0000 (Thu, 30 Oct 2008) $ # $Author: msteffens $ # $Revision: 1289 $ # REFBASE_IMPORT -- a refbase command line interface # Purpose: Perl script that allows to upload data in various formats to a refbase online database from the command line # Usage: refbase_import [OPTIONS] [FILE] # Help: For help with the syntax type 'refbase_import -h' # To view some usage examples type 'refbase_import -X' # Further information is available at # A list of supported import formats is given at # Version: 1.2 # Requires: - a shell with Perl execution capabilities # - the Perl CPAN modules LWP::UserAgent, HTTP::Request::Common, HTTP::Response, HTTP::Cookies and URI::URL # - access with import permissions to a refbase database (refbase-0.9.0 or greater) # Limits: - The character encoding of your import data must match the encoding of your refbase database (i.e., 'latin1' or 'utf8') # - The authentication mechanism is currently limited in that a given password will be transferred as parameter in the POST request # -------------------------------------------------------------------------------------------------------------- $version = "1.2"; # Configure variables: # Specify the full URLs to any refbase servers that shall be queried: # Notes: - the given hash keys will work as shortcuts, e.g. '--host=local' would upload # data to your local refbase installation; one hash key must be named 'default', # all other keys can be freely chosen # - by default, data will be uploaded to the server labeled with key 'default' %hosts = ( 'default' => 'http://beta.refbase.net/', 'local' => 'http://localhost/refs/', 'beta' => 'http://beta.refbase.net/', 'beta2' => 'http://refbase.textdriven.com/beta/', 'demo' => 'http://demo.refbase.net/', 'org' => 'http://www.refbase.org/' ); # Specify the default values for all options that are not explicitly specified: %params = ( # import options: 'skipBadRecords' => '0', # -b|--skipbad -> must be '0' (don't skip records with unrecognized data format) or '1' (skip records with unrecognized data format) 'importRecordsRadio' => 'all', # -i|--import -> must be 'all' (import all records) or 'only' (import only those records specified in 'importRecords') 'sourceIDs' => '', # -p|--pmid -> this also applies for '--arxiv|--doi|--openurl' since they are essentially just aliases for '-p' 'importRecords' => '1', # -r|--records -> must be a list of numbers and/or ranges (e.g., '1-5' will import the first five records; '1 3-5 7' will import records 1, 3, 4, 5 and 7) 'formType' => 'data', # -t|--type -> must be 'data' (generic data import) or 'id' (import via ID) # fixed parameters: 'client' => "cli-refbase_import-" . $version # the client ID of this command line utility ); %outputParams = ( # output options: 'citeStyle' => '', # -C|--style => desired citation style, given name must match an entry within the database's MySQL table 'styles' (keep empty to use the database default) 'format' => 'ascii', # -F|--format => output format must be 'html', 'rtf', 'pdf', 'latex', 'latex_bbl', 'markdown', 'ascii', 'ads', 'bibtex', 'endnote', 'isi', 'ris', 'atom', 'mods', 'oai_dc', 'odf', 'srw_dc', 'srw_mods', 'word' or '' (the empty string '' will produce the default 'ascii' output style) 'showLinks' => '1', # -L|--showlinks => hide/display links column in HTML output; must be '0', '1', or '' (the empty string '' will produce the default output style, i.e. print any links) 'citeOrder' => 'author', # -O|--order => cite order must be 'author', 'year', 'type', 'type-year', 'creation-date' or '' (the empty string '' will produce the default 'author' sort order) 'viewType' => 'web' # -V|--view => view type of HTML output; must be 'Web', 'Print', 'Mobile' or '' (the empty string '' will produce the default 'Web' output style) ); # Specify the default login credentials for a refbase user account: # Imported data will get associated with this user account %loginParams = ( 'loginEmail' => '', # -U|--user -> the login email address of an existing refbase user with import permissions 'loginPassword' => '' # -P|--password -> the password for the given user account ); # Specify the location of the cookie jar file: # This file will be used to store & retrieve cookies $cookieJarFile = "$ENV{HOME}/.lwpcookies.txt"; # -------------------------------------------------------------------------------- # NOTE: You shouldn't need to change anything below this line # CPAN modules: use LWP::UserAgent; # more info: use HTTP::Request::Common; # more info: use HTTP::Response; # more info: use HTTP::Cookies; # more info: use URI::URL; # more info: # initialize variables: $host = $hosts{'default'}; $format = ''; # Extract options: # TODO: use Getopt::Long # general options: if (($ARGV[0] eq '--help') or ($ARGV[0] eq '-h') or ($ARGV[0] eq '')) { &usage (0); } # if the user asked for --help/-h or didn't provide any input, call the 'usage' subroutine elsif (($ARGV[0] eq '--version') or ($ARGV[0] eq '-v')) { &version (0); } # show version information elsif (($ARGV[0] eq '--examples') or ($ARGV[0] eq '-X')) { &examples (0); } # print some usage examples else { foreach (@ARGV) { # extract import options: if ($_ =~ /^(?:-b|--skipbad)=(.+)$/) { $params{'skipBadRecords'} = $1; } elsif ($_ =~ /^(?:-i|--import)=(.+)$/) { $params{'importRecordsRadio'} = $1; } elsif ($_ =~ /^(?:-p|--pmid|--arxiv|--doi|--openurl)=(.+)$/) { $params{'sourceIDs'} = $1; } elsif ($_ =~ /^(?:-r|--records)=(.+)$/) { $params{'importRecords'} = $1; } elsif ($_ =~ /^(?:-t|--type)=(.+)$/) { $params{'formType'} = $1; } # extract output options: elsif ($_ =~ /^(?:-C|--style)=(.+)$/) { $outputParams{'citeStyle'} = $1; } elsif ($_ =~ /^(?:-F|--format)=(.+)$/) { $outputParams{'format'} = $1; } elsif ($_ =~ /^(?:-L|--showlinks)=(.+)$/) { $outputParams{'showLinks'} = $1; } elsif ($_ =~ /^(?:-O|--order)=(.+)$/) { $outputParams{'citeOrder'} = $1; } elsif ($_ =~ /^(?:-V|--view)=(.+)$/) { $outputParams{'viewType'} = $1; } # extract server options: elsif ($_ =~ /^(?:-H|--host)=(.+)$/) { $host = $1; } elsif ($_ =~ /^(?:-P|--password)=(.+)$/) { $loginParams{'loginPassword'} = $1; } elsif ($_ =~ /^(?:-U|--user)=(.+)$/) { $loginParams{'loginEmail'} = $1; } # extract file: # (note that if multiple files were given, only the last given file will be honoured) elsif ($_ =~ /^(?!(-[biprtCFLOVHPU]|--(?:skipbad|import|pmid|arxiv|doi|openurl|records|type|style|format|showlinks|order|view|host|password|user))=)([^ ]+)/) { @sourceFile = $2; } } } # for '--type=data', check if a source file was specified: if (($params{'formType'} =~ /^data$/i) && (scalar @sourceFile == 0)) { print "There were validation errors regarding the data you submitted:\n\n"; print "FILE: The file operand is missing! The generic data import feature ('--type=data')\n" . " requires a FILE to be specified. Type 'refbase_import -X' to see some usage\n" . " examples. For general help with the syntax type 'refbase_import -h'.\n\n"; exit; } # for '--type=id' (or, previously: --type=pmid), check if at least one PubMed ID, arXiv ID, DOI or OpenURL was given: # TODO: improve identification/verification of the given IDs elsif (($params{'formType'} =~ /^(pm)?id$/i) && ($params{'sourceIDs'} !~ /\d+/)) { print "There were validation errors regarding the data you submitted:\n\n"; print "sourceIDs: You must specify at least one PubMed ID, arXiv ID, DOI or OpenURL! The 'import via ID'\n" . " feature ('--type=id') requires the '-p, --pmid' option or one of '--arxiv|--doi|--openurl'\n" . " to be specified. Type 'refbase_import -X' to see some usage examples. For general help\n" . " with the syntax type 'refbase_import -h'.\n\n"; exit; } # adjust form type value: if ($params{'formType'} =~ /^(pm)?id$/i) { # --type=id (or, previously: --type=pmid) $params{'formType'} = "importID"; } else { # --type=data $params{'formType'} = "import"; } # resolve any host shortcuts: if (exists($hosts{$host})) { $host = $hosts{$host}; } elsif ($host !~ /^https?:\/\//i) { $host = $hosts{'default'}; # can't resolve given host, reset back to default } # assign correct URL params based on the '-F|--format' option: if (exists($outputParams{'format'})) { $format = $outputParams{'format'}; if ($format =~ /^(html|rtf|pdf|latex|latex_bbl|markdown|ascii)$/i) { $outputParams{'submit'} = "Cite"; } if ($format =~ /^(html|rtf|pdf|latex|latex_bbl|markdown|ascii)$/i) { $format =~ s/^latex_bbl$/LaTeX .bbl/i; $outputParams{'citeType'} = $format; } elsif ($format =~ /^(ads|bibtex|endnote|isi|ris|atom|mods|oai_dc|odf|srw(_dc|_mods)?|word)$/i) { $outputParams{'submit'} = "Export"; $outputParams{'exportType'} = "file"; if ($format =~ /^(ads|bibtex|endnote|isi|ris)$/i) { $outputParams{'exportFormat'} = $format; } elsif ($format =~ /^(atom|mods|oai_dc|odf|srw(_dc|_mods)?|word)$/i) { $outputParams{'exportFormat'} = $format . " xml"; } } else { $outputParams{'citeType'} = "ascii"; } delete($outputParams{'format'}); } # construct URL: # (uses URI::URL) $importScript = "import_modify.php"; $importURL = url($host . $importScript); # initialize new user agent: # (uses LWP::UserAgent) $userAgent = LWP::UserAgent->new; # set user agent string: $userAgent->agent("refbase_import/" . $version . " (http://cli.refbase.net/) "); # set cookie jar object: # LWP will collect cookies and respond to cookie requests via its cookie jar, thus # enabling the user agent to fetch a PHP session ID from the refbase login response # and automatically resend it upon next import request $userAgent->cookie_jar({ file => $cookieJarFile, autosave => 1 }); # attempt to authenticate using the given login credentials: if (($loginParams{'loginEmail'} ne '') && ($loginParams{'loginPassword'} ne '')) { $loginSuccessful = &login(0); # call the 'login' subroutine } else { $loginSuccessful = 0; } if (!$loginSuccessful) { print "Login failed! You provided an incorrect email address or password.\n\n"; exit; } # send POST request: # (uses HTTP::Request::Common & HTTP::Response) if ($params{'formType'} =~ /^importID$/i) { # --type=id (or, previously: --type=pmid) $request = POST $importURL, \%params; } else { # --type=data $params{'uploadFile'} = \@sourceFile; $request = POST $importURL, Content_Type => 'form-data', Content => \%params; } $response = $userAgent->request($request); if ($response->is_error()) { print STDERR $response->status_line, "\n"; } else { $location = $response->header('Location'); if ($location ne '') { if ($location =~ /show.php/) { # display imported records: foreach $key (keys %outputParams) { $location .= "&" . $key . "=" . $outputParams{$key}; } if ($location =~ /&headerMsg=\D*(\d+)/i) { $location .= "&showRows=" . $1; } } # construct URL: # (uses URI::URL) $responseURL = url($host . $location); # send GET request: # (uses HTTP::Request::Common & HTTP::Response) $request = GET $responseURL; $response = $userAgent->request($request); # or use: $response = $userAgent->get($responseURL); } binmode STDOUT; print $response->content(); } # -------------------------------------------------------------------------------- # Login with login credentials given in '%loginParams': sub login { local ($status) = @_; # construct URL: # (uses URI::URL) $loginScript = "user_login.php"; $loginURL = url($host . $loginScript); # send POST request: # (uses HTTP::Request::Common & HTTP::Response) $request = POST $loginURL, \%loginParams; $response = $userAgent->request($request); if ($response->is_error()) { print STDERR $response->status_line, "\n"; exit $status; } else { $location = $response->header('Location'); # upon successful login, refbase will redirect to 'index.php' if ($location =~ /index.php/) { return 1; # login successful } else { return 0; # login NOT successful } } } # -------------------------------------------------------------------------------- # Print usage and exit: sub usage { local ($status) = @_; print "\nrefbase_import command line client, v" . $version . " by Matthias Steffens, http://cli.refbase.net/\n\n" . "Usage: refbase_import [OPTIONS] [FILE]\n\n" . "Notes: - Two import modes are supported:\n" . " 1) '--type=data' requires an import FILE to be specified;\n" . " for supported import formats, see: http://import.refbase.net/\n" . " 2) '--type=id' requires the '-p, --pmid' option or one of '--arxiv|--doi|--openurl' with\n" . " one or more whitespace-delimited PubMed IDs, arXiv IDs, DOIs or OpenURLs, respectively.\n" . " - Options syntax: [OPTION]=[VALUE], e.g. '-p=16351846' or '--pmid=\"16351846 16783713\"'.\n" . " - For each option, default values can be specified at the top of the script.\n" . " Current defaults are given in parentheses.\n\n" . "General Options: -h, --help - display this help text\n" . " -v, --version - display version information\n" . " -X, --examples - display usage examples\n\n" . "Import Options: -b, --skipbad - skip records with unrecognized data format ('" . $params{'skipBadRecords'} . "')\n" . " possible values: 0, 1\n" . " -i, --import - import all or only some records ('" . $params{'importRecordsRadio'} . "')\n" . " possible values: all, only\n" . " -p, --pmid, - IDs of records to import ('" . $params{'sourceIDs'} . "')\n" . " --arxiv, supported IDs: PubMed ID (PMID), arXiv ID, DOI, OpenURL\n" . " --doi, \n" . " --openurl \n" . " -r, --records - positional numbers and/or ranges of records to import ('" . $params{'importRecords'} . "')\n" . " requires the '--import=only' option\n" . " -t, --type - import type ('" . $params{'formType'} . "')\n" . " possible values: data, id\n\n" . "Output Options: -C, --style - citation style ('" . $outputParams{'citeStyle'} . "')\n" . " -F, --format - output format ('" . $outputParams{'format'} . "')\n" . " possible values: html, rtf, pdf, latex, latex_bbl, markdown, ascii,\n" . " ads, bibtex, endnote, isi, ris, atom, mods, oai_dc,\n" . " odf, srw_dc, srw_mods, word\n" . " -L, --showlinks - hide/display links column in html output ('" . $outputParams{'showLinks'} . "')\n" . " possible values: 0, 1\n" . " -O, --order - sort order of returned records ('" . $outputParams{'citeOrder'} . "')\n" . " possible values: author, year, type, type-year, creation-date\n" . " -V, --view - view type of html output ('" . $outputParams{'viewType'} . "')\n" . " possible values: web, print, mobile\n\n" . "Server Options: -H, --host - URL of the refbase database ('" . $host . "')\n" . " defined shortcuts: " . join(', ', sort keys(%hosts)) . "\n" . " -P, --password - password for given '-U, --user' account"; if ($loginParams{'loginPassword'} ne '') { print "\n (a default pwd has been defined)\n"; } else { print " ('')\n"; } print " -U, --user - login email address of an existing refbase user with\n" . " import permissions ('" . $loginParams{'loginEmail'} . "')\n\n"; exit $status; } # -------------------------------------------------------------------------------- # Print version number and exit: sub version { local ($status) = @_; print "\nrefbase_import command line client, version " . $version . "\ncheck for updates at http://cli.refbase.net/\n\n"; exit $status; } # -------------------------------------------------------------------------------- # Print examples and exit: sub examples { local ($status) = @_; print <<'END_EXAMPLES'; -------------------------------------------------------------------------------- REFBASE_IMPORT USAGE EXAMPLES: -------------------------------------------------------------------------------- 1) Import BibTeX records from file 'import.bib' using the defaults defined within the refbase_import script: refbase_import import.bib -------------------------------------------------------------------------------- 2) Import all Endnote records given in file 'import.enw' into your default refbase database: refbase_import -t=data -i=all import.enw -------------------------------------------------------------------------------- 3) Take RIS records from file 'import.ris' but import only the first three as well as the fifth and the tenth record into your local refbase database: refbase_import -H=local -t=data -i=only -r=1-3,5,10 import.ris -------------------------------------------------------------------------------- 4) Import MODS XML records from file 'mods.xml' into the refbase demo database using the defaults defined within the refbase_import script: refbase_import -H=http://demo.refbase.net/ -U=user@refbase.net -P=user mods.xml -------------------------------------------------------------------------------- 5) Fetch two records from PubMed.gov via their PMID (i.e. the unique PubMed identifier, in this example, records with PMIDs 16351846 and 16783713) and import them into your local refbase database: refbase_import -H=local -t=id -p="16351846 16783713" -------------------------------------------------------------------------------- 6) Fetch two records from CrossRef.org via their DOI (i.e. the unique Document Object Identifier, in this example, records with DOIs 10.3354/meps251037 and 10.1103/PhysRev.47.777) and import them into your local refbase database: refbase_import -H=local -t=id --doi="10.3354/meps251037 10.1103/PhysRev.47.777" -------------------------------------------------------------------------------- 7) Fetch three records from arXiv.org via their arXiv ID (i.e. the unique arXiv identifier, in this example, records with arXiv IDs astro-ph/0609768, 0806.1829 and 0802.0204v1) and import them into your local refbase database: refbase_import -H=local -t=id --arxiv="astro-ph/0609768 0806.1829 0802.0204v1" -------------------------------------------------------------------------------- END_EXAMPLES exit $status; } __END__