LoginSignup
1
3

More than 5 years have passed since last update.

@revogue

Amazon.comの500アイテムとAmazon.co.jpのアイテムの価格を一挙に比較するプログラム（パート２）

Last updated at Posted at 2016-09-16

パート２です。

Amazon.comのIDと秘密鍵が必要です。
さらに、Amazon.comのアソシエイトタグとAmazon.co.jpのアソシエイトタグが必要です。
（Amazon海外輸出などの本を参考にして、アソシエイトタグを入手して下さい。）

Amazon.comへのアクセスは１秒に１回にしています。ときどき怒られるときがあるので、その場合は、２倍の時間Sleepさせてから再度アクセスします。
Amazon.co.jpへのアクセスは５秒に１回にしています。こちらも怒られたら、２倍の時間Sleepさせてから再度アクセスします。
出力結果は、STDOUTにでてきます。

あと、wgetを使っています。（コーディングが下手ですみません。）

パート２は以上です。

comp_us_jp.pl
#!/usr/bin/perl -w

use strict;
use warnings;

use Data::Dumper;

use RequestSignatureHelper;
use LWP::UserAgent;
use XML::Simple;

use constant myAWSId        => ''; //AWSのIDを入れて下さい
use constant myAWSSecret    => ''; //AWSの秘密鍵を入れて下さい
use constant myEndPoint     => 'ecs.amazonaws.com';
use constant jmyEndPoint        => 'webservices.amazon.co.jp';

# see if user provided ItemId on command-line
my $itemId = shift @ARGV || 'B00006JNN7';

# Set up the helper
my $helper = new RequestSignatureHelper (
    +RequestSignatureHelper::kAWSAccessKeyId => myAWSId,
    +RequestSignatureHelper::kAWSSecretKey => myAWSSecret,
    +RequestSignatureHelper::kEndPoint => myEndPoint,
);

my %asin_hash=();
my %line_hash=();

#
# input data
#
while(my $line = <STDIN> )
{
    chomp( $line );
        (my $asin ) = ($line =~ /([A-Z0-9]{10}).*/);
    $line_hash{ $asin } = $line;
    $asin_hash{ $asin } = $asin;
}

#
# make query lines
#
my %query_lines_hash=();
my $asin_counter_per_line = 0;
my $query_line = "";

foreach my $key (keys(%asin_hash))
{
    $query_line = $query_line . "," . $key ;
    $asin_counter_per_line ++ ;
    if($asin_counter_per_line eq 10)
    {
        $query_lines_hash{ $query_line } = $query_line ;
        $query_line = "";
        $asin_counter_per_line = 0 ;
    }
}
if( $asin_counter_per_line ne 0 ) 
{
    $query_lines_hash{ $query_line } = $query_line ;
    $query_line = "";
    $asin_counter_per_line = 0 ;
}

my %rank_hash=();
my %price_hash=();
my %type_hash=();
my %weight_hash=();
my %url_hash=();
my %offers_url_hash=();
my %sutable_price_hash=();

my $base_sleep_time = 1;

foreach my $key (keys(%query_lines_hash)){
    my $sleep_time = $base_sleep_time ;
    my $error_count = 0;

RETURN_POINT:
    my $request = {
        Service => 'AWSECommerceService',
        Operation => 'ItemLookup',
        AssociateTag  => '', //amazon.comのアソシエイトタグを入れて下さい
        ResponseGroup => 'SalesRank,OfferSummary,OfferFull,ItemAttributes',
        ItemId => $key,
    };

    # Sign the request
    my $signedRequest = $helper->sign($request);

    # We can use the helper's canonicalize() function to construct the query string too.
    my $queryString = $helper->canonicalize($signedRequest);
    my $url = "http://" . myEndPoint . "/onca/xml?" . $queryString;

    my $ua = new LWP::UserAgent();
    my $response = $ua->get($url);
    my $content = $response->content();

    my $xmlParser = new XML::Simple();
    my $xml = $xmlParser->XMLin($content,ForceArray=>['Item']);

    if( exists $xml->{Items}{Item} && @{ $xml->{Items}{Item} } ) {
        foreach ( @{ $xml->{Items}{Item} } )
        {
            #print "A=" . $_->{ASIN} . "\n" ;

            $rank_hash{ $_->{ASIN} } = $_->{SalesRank} ;
            $price_hash{ $_->{ASIN} } = $_->{OfferSummary}{LowestNewPrice}{Amount};
            $type_hash{ $_->{ASIN} } = $_->{ItemAttributes}{ProductGroup};

            if( exists $_->{DetailPageURL} ) {
                $url_hash{ $_->{ASIN} } = $_->{DetailPageURL} ;
            }

            if( exists $_->{ItemAttributes} ) {
                if( exists $_->{ItemAttributes}{PackageDimensions}{Weight}{content}) {
                    $weight_hash{ $_->{ASIN} } = $_->{ItemAttributes}{PackageDimensions}{Weight}{content} ;
                }
            }
            if( exists $_->{Offers}{MoreOffersUrl} ) {
                $offers_url_hash{ $_->{ASIN} } = $_->{Offers}{MoreOffersUrl} ;
                $offers_url_hash{ $_->{ASIN} } =~ s/^http/https/ ;
                #print $_->{ASIN} ."---". $offers_url_hash{ $_->{ASIN} }."\n" ;
            }
        }
    }

    if ($response->is_success()) {
    } else {
            my $error = findError($xml);
            if (defined $error) {
            print STDERR "Error: " . $error->{Code} . ": " . $error->{Message} . "\n";
            if( $error_count > 2 ){ if($error_count eq 3) {print "DATA_LOST\n";} }else
            { 
                $error_count++; 
                print STDERR "ERROR_COUNT=$error_count\n"; 
                $sleep_time = $sleep_time * 2; 
                print STDERR "SLEEP_TIME=$sleep_time\n";
                sleep $sleep_time;
                goto RETURN_POINT ;
            }
            } else {
            print "Unknown Error!\n";
            }
    }
    sleep $sleep_time;
}

#
# sutable_price
#

my $usd_head = 0;
my $usd_tail = 0;

foreach my $key (keys(%offers_url_hash)){
    #print $offers_url_hash{ $key } . "\n";
    my $command = "/usr/local/bin/wget -q -O - ".$offers_url_hash{ $key }." | grep 'a-color-price' > aoutwget.html" ;
    system($command);
    $command = "/usr/bin/head -n 1 < aoutwget.html > aoutwget_head.html" ;
    system($command);
    $command = "/usr/bin/tail -n 1 < aoutwget.html > aoutwget_tail.html" ;
    system($command);

    open(my $fh, "<", "aoutwget_head.html") or die "Cannot open aoutwget_head.html";
    while(my $line = <$fh>){ 
        chomp $line;
            (my $usd_tmp) = ($line =~ /(\$[\.0-9]*)/);
        $usd_tmp =~ s/^.// ;
        $usd_tmp =~ s/\.// ;
        $usd_head = $usd_tmp + 0;
    }
    close($fh);
    open(   $fh, "<", "aoutwget_tail.html") or die "Cannot open aoutwget_tail.html";
    while(my $line = <$fh>){ 
        chomp $line;
            (my $usd_tmp) = ($line =~ /(\$[\.0-9]*)/);
        $usd_tmp =~ s/^.// ;
        $usd_tmp =~ s/\.// ;
        $usd_tail = $usd_tmp + 0;
    }
    close($fh);
    $sutable_price_hash{ $key } =int(($usd_head+$usd_tail)/2+0.5);
}

my %jrank_hash=();
my %jprice_hash=();
my %jurl_hash=();
my %prime_hash=();
my %totalnew_hash=();


# Set up the helper
$helper = new RequestSignatureHelper (
    +RequestSignatureHelper::kAWSAccessKeyId => myAWSId,
    +RequestSignatureHelper::kAWSSecretKey => myAWSSecret,
    +RequestSignatureHelper::kEndPoint => jmyEndPoint,
);

$base_sleep_time = 5;

foreach my $key (keys(%query_lines_hash)){
    my $sleep_time = $base_sleep_time ;
    my $error_count = 0;

RETURN_POINT2:

    my $request = {
        Service => 'AWSECommerceService',
        Operation => 'ItemLookup',
        AssociateTag  => '', //amazon.co.jpのアソシエイトタグを入れて下さい
        ItemId => $key,
        ResponseGroup => 'ItemIds,SalesRank,Offers,OfferSummary,ItemAttributes,Large',
    };

    # Sign the request
    my $signedRequest = $helper->sign($request);

    # We can use the helper's canonicalize() function to construct the query string too.
    my $queryString = $helper->canonicalize($signedRequest);
    my $url = "http://" . jmyEndPoint . "/onca/xml?" . $queryString;

    my $ua = new LWP::UserAgent();
    my $response = $ua->get($url);
    my $content = $response->content();

    my $xmlParser = new XML::Simple();
    my $xml = $xmlParser->XMLin($content,ForceArray=>['Item']);

    if( exists $xml->{Items}{Item} && @{ $xml->{Items}{Item} } ) {
        foreach ( @{ $xml->{Items}{Item} } )
        {
            if( exists $_->{SalesRank} ) {
                $jrank_hash{ $_->{ASIN} } = $_->{SalesRank} ;
            }

            if( exists $_->{OfferSummary}{LowestNewPrice}{Amount}  )
            {
                    $jprice_hash{ $_->{ASIN} } = $_->{OfferSummary}{LowestNewPrice}{Amount} ;
            }
            if( exists $_->{DetailPageURL} ) {
                $jurl_hash{ $_->{ASIN} } = $_->{DetailPageURL} ;
            }
            if( exists $_->{Offers}{Offer}  && exists $_->{Offers}{Offer}{OfferListing}{IsEligibleForPrime}  )
            {
                $prime_hash{ $_->{ASIN} } = $_->{Offers}{Offer}{OfferListing}{IsEligibleForPrime};
            }
            if( exists $_->{OfferSummary}{TotalNew}  )
            {
                    $totalnew_hash{ $_->{ASIN} } = $_->{OfferSummary}{TotalNew} ;
            }
        }
    }

    if ($response->is_success()) {
    } else {
            my $error = findError($xml);
            if (defined $error) {
            print STDERR "Error: " . $error->{Code} . ": " . $error->{Message} . "\n";
            if( $error_count > 2 ){ if($error_count eq 3) {print "DATA_LOST\n";} }else
            { 
                $error_count++; 
                print STDERR "ERROR_COUNT=$error_count\n"; 
                $sleep_time = $sleep_time * 2; 
                print STDERR "SLEEP_TIME=$sleep_time\n";
                sleep $sleep_time;
                goto RETURN_POINT2 ;
            }
            } else {
            print "Unknown Error!\n";
            }
    }
    sleep $sleep_time;
}

my %diff_hash=();

foreach my $key (keys(%asin_hash)){
    if( exists $price_hash{$key} )
    {
        $diff_hash{$key}=$price_hash{$key}-$jprice_hash{$key};
    }
    else
    {
        $diff_hash{$key}=0-$jprice_hash{$key};
    }
    print "$key\t$jrank_hash{$key}\t$jprice_hash{$key}\t";
    if( exists $rank_hash{$key} ) { print $rank_hash{$key} ; }
    print "\t";
    if( exists $price_hash{$key} ) { print $price_hash{$key} ; }
    print "\t$jurl_hash{$key}\t$url_hash{$key}\t$weight_hash{$key}\t$diff_hash{$key}\t$prime_hash{$key}\t$sutable_price_hash{$key}\t$totalnew_hash{$key}\n";
}

sub findError {
    my $xml = shift;

    return undef unless ref($xml) eq 'HASH';

    if (exists $xml->{Error}) { return $xml->{Error}; };

    for (keys %$xml) {
    my $error = findError($xml->{$_});
    return $error if defined $error;
    }

    return undef;
}

Perlソースコード終わり

1
3
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
1
3