Help us understand the problem. What is going on with this article?

Amazon.comの500アイテムとAmazon.co.jpのアイテムの価格を一挙に比較するプログラム(パート2)

More than 3 years have passed since last update.

パート2です。

Amazon.comのIDと秘密鍵が必要です。
さらに、Amazon.comのアソシエイトタグとAmazon.co.jpのアソシエイトタグが必要です。
(Amazon海外輸出などの本を参考にして、アソシエイトタグを入手して下さい。)

Amazon.comへのアクセスは1秒に1回にしています。ときどき怒られるときがあるので、その場合は、2倍の時間Sleepさせてから再度アクセスします。
Amazon.co.jpへのアクセスは5秒に1回にしています。こちらも怒られたら、2倍の時間Sleepさせてから再度アクセスします。
出力結果は、STDOUTにでてきます。

あと、wgetを使っています。(コーディングが下手ですみません。)

パート2は以上です。

comp_us_jp.pl
#!/usr/bin/perl -w

use strict;
use warnings;

use Data::Dumper;

use RequestSignatureHelper;
use LWP::UserAgent;
use XML::Simple;

use constant myAWSId        => ''; //AWSIDを入れて下さい
use constant myAWSSecret    => ''; //AWSの秘密鍵を入れて下さい
use constant myEndPoint     => 'ecs.amazonaws.com';
use constant jmyEndPoint        => 'webservices.amazon.co.jp';

# see if user provided ItemId on command-line
my $itemId = shift @ARGV || 'B00006JNN7';

# Set up the helper
my $helper = new RequestSignatureHelper (
    +RequestSignatureHelper::kAWSAccessKeyId => myAWSId,
    +RequestSignatureHelper::kAWSSecretKey => myAWSSecret,
    +RequestSignatureHelper::kEndPoint => myEndPoint,
);

my %asin_hash=();
my %line_hash=();

#
# input data
#
while(my $line = <STDIN> )
{
    chomp( $line );
        (my $asin ) = ($line =~ /([A-Z0-9]{10}).*/);
    $line_hash{ $asin } = $line;
    $asin_hash{ $asin } = $asin;
}

#
# make query lines
#
my %query_lines_hash=();
my $asin_counter_per_line = 0;
my $query_line = "";

foreach my $key (keys(%asin_hash))
{
    $query_line = $query_line . "," . $key ;
    $asin_counter_per_line ++ ;
    if($asin_counter_per_line eq 10)
    {
        $query_lines_hash{ $query_line } = $query_line ;
        $query_line = "";
        $asin_counter_per_line = 0 ;
    }
}
if( $asin_counter_per_line ne 0 ) 
{
    $query_lines_hash{ $query_line } = $query_line ;
    $query_line = "";
    $asin_counter_per_line = 0 ;
}

my %rank_hash=();
my %price_hash=();
my %type_hash=();
my %weight_hash=();
my %url_hash=();
my %offers_url_hash=();
my %sutable_price_hash=();

my $base_sleep_time = 1;

foreach my $key (keys(%query_lines_hash)){
    my $sleep_time = $base_sleep_time ;
    my $error_count = 0;

RETURN_POINT:
    my $request = {
        Service => 'AWSECommerceService',
        Operation => 'ItemLookup',
        AssociateTag  => '', //amazon.comのアソシエイトタグを入れて下さい
        ResponseGroup => 'SalesRank,OfferSummary,OfferFull,ItemAttributes',
        ItemId => $key,
    };

    # Sign the request
    my $signedRequest = $helper->sign($request);

    # We can use the helper's canonicalize() function to construct the query string too.
    my $queryString = $helper->canonicalize($signedRequest);
    my $url = "http://" . myEndPoint . "/onca/xml?" . $queryString;

    my $ua = new LWP::UserAgent();
    my $response = $ua->get($url);
    my $content = $response->content();

    my $xmlParser = new XML::Simple();
    my $xml = $xmlParser->XMLin($content,ForceArray=>['Item']);

    if( exists $xml->{Items}{Item} && @{ $xml->{Items}{Item} } ) {
        foreach ( @{ $xml->{Items}{Item} } )
        {
            #print "A=" . $_->{ASIN} . "\n" ;

            $rank_hash{ $_->{ASIN} } = $_->{SalesRank} ;
            $price_hash{ $_->{ASIN} } = $_->{OfferSummary}{LowestNewPrice}{Amount};
            $type_hash{ $_->{ASIN} } = $_->{ItemAttributes}{ProductGroup};

            if( exists $_->{DetailPageURL} ) {
                $url_hash{ $_->{ASIN} } = $_->{DetailPageURL} ;
            }

            if( exists $_->{ItemAttributes} ) {
                if( exists $_->{ItemAttributes}{PackageDimensions}{Weight}{content}) {
                    $weight_hash{ $_->{ASIN} } = $_->{ItemAttributes}{PackageDimensions}{Weight}{content} ;
                }
            }
            if( exists $_->{Offers}{MoreOffersUrl} ) {
                $offers_url_hash{ $_->{ASIN} } = $_->{Offers}{MoreOffersUrl} ;
                $offers_url_hash{ $_->{ASIN} } =~ s/^http/https/ ;
                #print $_->{ASIN} ."---". $offers_url_hash{ $_->{ASIN} }."\n" ;
            }
        }
    }

    if ($response->is_success()) {
    } else {
            my $error = findError($xml);
            if (defined $error) {
            print STDERR "Error: " . $error->{Code} . ": " . $error->{Message} . "\n";
            if( $error_count > 2 ){ if($error_count eq 3) {print "DATA_LOST\n";} }else
            { 
                $error_count++; 
                print STDERR "ERROR_COUNT=$error_count\n"; 
                $sleep_time = $sleep_time * 2; 
                print STDERR "SLEEP_TIME=$sleep_time\n";
                sleep $sleep_time;
                goto RETURN_POINT ;
            }
            } else {
            print "Unknown Error!\n";
            }
    }
    sleep $sleep_time;
}

#
# sutable_price
#

my $usd_head = 0;
my $usd_tail = 0;

foreach my $key (keys(%offers_url_hash)){
    #print $offers_url_hash{ $key } . "\n";
    my $command = "/usr/local/bin/wget -q -O - ".$offers_url_hash{ $key }." | grep 'a-color-price' > aoutwget.html" ;
    system($command);
    $command = "/usr/bin/head -n 1 < aoutwget.html > aoutwget_head.html" ;
    system($command);
    $command = "/usr/bin/tail -n 1 < aoutwget.html > aoutwget_tail.html" ;
    system($command);

    open(my $fh, "<", "aoutwget_head.html") or die "Cannot open aoutwget_head.html";
    while(my $line = <$fh>){ 
        chomp $line;
            (my $usd_tmp) = ($line =~ /(\$[\.0-9]*)/);
        $usd_tmp =~ s/^.// ;
        $usd_tmp =~ s/\.// ;
        $usd_head = $usd_tmp + 0;
    }
    close($fh);
    open(   $fh, "<", "aoutwget_tail.html") or die "Cannot open aoutwget_tail.html";
    while(my $line = <$fh>){ 
        chomp $line;
            (my $usd_tmp) = ($line =~ /(\$[\.0-9]*)/);
        $usd_tmp =~ s/^.// ;
        $usd_tmp =~ s/\.// ;
        $usd_tail = $usd_tmp + 0;
    }
    close($fh);
    $sutable_price_hash{ $key } =int(($usd_head+$usd_tail)/2+0.5);
}

my %jrank_hash=();
my %jprice_hash=();
my %jurl_hash=();
my %prime_hash=();
my %totalnew_hash=();


# Set up the helper
$helper = new RequestSignatureHelper (
    +RequestSignatureHelper::kAWSAccessKeyId => myAWSId,
    +RequestSignatureHelper::kAWSSecretKey => myAWSSecret,
    +RequestSignatureHelper::kEndPoint => jmyEndPoint,
);

$base_sleep_time = 5;

foreach my $key (keys(%query_lines_hash)){
    my $sleep_time = $base_sleep_time ;
    my $error_count = 0;

RETURN_POINT2:

    my $request = {
        Service => 'AWSECommerceService',
        Operation => 'ItemLookup',
        AssociateTag  => '', //amazon.co.jpのアソシエイトタグを入れて下さい
        ItemId => $key,
        ResponseGroup => 'ItemIds,SalesRank,Offers,OfferSummary,ItemAttributes,Large',
    };

    # Sign the request
    my $signedRequest = $helper->sign($request);

    # We can use the helper's canonicalize() function to construct the query string too.
    my $queryString = $helper->canonicalize($signedRequest);
    my $url = "http://" . jmyEndPoint . "/onca/xml?" . $queryString;

    my $ua = new LWP::UserAgent();
    my $response = $ua->get($url);
    my $content = $response->content();

    my $xmlParser = new XML::Simple();
    my $xml = $xmlParser->XMLin($content,ForceArray=>['Item']);

    if( exists $xml->{Items}{Item} && @{ $xml->{Items}{Item} } ) {
        foreach ( @{ $xml->{Items}{Item} } )
        {
            if( exists $_->{SalesRank} ) {
                $jrank_hash{ $_->{ASIN} } = $_->{SalesRank} ;
            }

            if( exists $_->{OfferSummary}{LowestNewPrice}{Amount}  )
            {
                    $jprice_hash{ $_->{ASIN} } = $_->{OfferSummary}{LowestNewPrice}{Amount} ;
            }
            if( exists $_->{DetailPageURL} ) {
                $jurl_hash{ $_->{ASIN} } = $_->{DetailPageURL} ;
            }
            if( exists $_->{Offers}{Offer}  && exists $_->{Offers}{Offer}{OfferListing}{IsEligibleForPrime}  )
            {
                $prime_hash{ $_->{ASIN} } = $_->{Offers}{Offer}{OfferListing}{IsEligibleForPrime};
            }
            if( exists $_->{OfferSummary}{TotalNew}  )
            {
                    $totalnew_hash{ $_->{ASIN} } = $_->{OfferSummary}{TotalNew} ;
            }
        }
    }

    if ($response->is_success()) {
    } else {
            my $error = findError($xml);
            if (defined $error) {
            print STDERR "Error: " . $error->{Code} . ": " . $error->{Message} . "\n";
            if( $error_count > 2 ){ if($error_count eq 3) {print "DATA_LOST\n";} }else
            { 
                $error_count++; 
                print STDERR "ERROR_COUNT=$error_count\n"; 
                $sleep_time = $sleep_time * 2; 
                print STDERR "SLEEP_TIME=$sleep_time\n";
                sleep $sleep_time;
                goto RETURN_POINT2 ;
            }
            } else {
            print "Unknown Error!\n";
            }
    }
    sleep $sleep_time;
}

my %diff_hash=();

foreach my $key (keys(%asin_hash)){
    if( exists $price_hash{$key} )
    {
        $diff_hash{$key}=$price_hash{$key}-$jprice_hash{$key};
    }
    else
    {
        $diff_hash{$key}=0-$jprice_hash{$key};
    }
    print "$key\t$jrank_hash{$key}\t$jprice_hash{$key}\t";
    if( exists $rank_hash{$key} ) { print $rank_hash{$key} ; }
    print "\t";
    if( exists $price_hash{$key} ) { print $price_hash{$key} ; }
    print "\t$jurl_hash{$key}\t$url_hash{$key}\t$weight_hash{$key}\t$diff_hash{$key}\t$prime_hash{$key}\t$sutable_price_hash{$key}\t$totalnew_hash{$key}\n";
}

sub findError {
    my $xml = shift;

    return undef unless ref($xml) eq 'HASH';

    if (exists $xml->{Error}) { return $xml->{Error}; };

    for (keys %$xml) {
    my $error = findError($xml->{$_});
    return $error if defined $error;
    }

    return undef;
}

Perlソースコード終わり

Why not register and get more from Qiita?
  1. We will deliver articles that match you
    By following users and tags, you can catch up information on technical fields that you are interested in as a whole
  2. you can read useful information later efficiently
    By "stocking" the articles you like, you can search right away
Comments
No comments
Sign up for free and join this conversation.
If you already have a Qiita account
Why do not you register as a user and use Qiita more conveniently?
You need to log in to use this function. Qiita can be used more conveniently after logging in.
You seem to be reading articles frequently this month. Qiita can be used more conveniently after logging in.
  1. We will deliver articles that match you
    By following users and tags, you can catch up information on technical fields that you are interested in as a whole
  2. you can read useful information later efficiently
    By "stocking" the articles you like, you can search right away
ユーザーは見つかりませんでした