我、日本人顔暗記開始了

# -*- coding: utf-8 -*-
import scrapy


class PasonicaSpider(scrapy.Spider):

    name = 'pasonica'
    allowed_domains = ['www.pasonica.com']
    download_delay = 1

    def start_requests(self):
        for year in range(2000, 1910, -10):
            for gender in ('女', '男'):
                url = 'https://www.pasonica.com/{}性芸能人-{}年代誕生/'.format(gender, year)
                yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        for div in response.css('.thumb_sq2'):
            for link in div.css('a'):
                profile_url = link.attrib.get('href')
                yield response.follow(url=profile_url, callback=self.parse_celebrity)

    def parse_celebrity(self, response):
        name = response.css('h2::text').get()
        image = response.css('.j-blog-post--header + div img').attrib.get('src', '')
        intro = ''.join(response.css('.j-module.n.j-text p::text').getall())
        self.log('GET {}'.format(name))
        return dict(name=name, image=image, intro=intro)

# coding: utf-8

import json
import re
import csv

RE_YOMI = re.compile('■(?P<name>.*)（(?P<yomi>.*)）') 
RE_NAME = re.compile('■(?P<name>.*)') 

def clean_intro(i):
    start_flag = '■'
    end_flag = '関連項目'
    start_idx = i.find(start_flag)
    if start_idx:
        i = i[start_idx:]
        end_idx = i.find(end_flag)
        if end_idx:
            i = i[:end_idx]
    return i.strip()

def get_name(i):
    m = RE_NAME.match(i)
    return m.group('name') if m else ''

def get_yomi(i):
    m = RE_YOMI.match(i)
    return m.group('yomi') if m else ''

def text_to_html(i):
    return i.replace('\n\n', '\n').replace('\n', '</br>')

def main():
    result = []
    with open('celebrity.json') as f:
        celebrities = json.load(f)
        for c in celebrities:
            intro = clean_intro(c['intro'])
            name = c['name']
            if not c['name']:
                name = get_name(intro)
            yomi = get_yomi(intro)
            if yomi or name:
                print(name, yomi)
                result.append(dict(name=name, yomi=yomi, intro=intro, image=c['image']))

    with open('celebrity_clean.json', 'w') as f:
        json.dump(result, f)

    with open('celebrity.csv', 'w') as f:
        csv_writer = csv.writer(f)
        for c in result:
            name = c['name']
            yomi = c['yomi']
            intro = text_to_html(c['intro'])
            row = name, yomi, '<img src="%s_%s.jpg" />' % (name, yomi), intro
            csv_writer.writerow(row)

if __name__ == '__main__':
    main()

package main

import (
    "encoding/json"
    "io/ioutil"
    "log"
    "net/http"
    "path/filepath"
    "sync"
)

type Celebrity struct {
    Name  string `json:"name"`
    Yomi  string `json:"yomi"`
    Image string `json:"image"`
}

var downloadDestFolder = "images"

func download(url string, filename string, w *sync.WaitGroup) {
    defer w.Done()
    res, err := http.Get(url)
    if err != nil {
        log.Printf("http.Get -> %v", err)
        return
    }
    data, err := ioutil.ReadAll(res.Body)
    if err != nil {
        log.Printf("ioutil.ReadAll -> %s", err.Error())
        return
    }
    defer res.Body.Close()
    if err = ioutil.WriteFile(downloadDestFolder+string(filepath.Separator)+filename, data, 0644); err != nil {
        log.Println("Error Saving:", filename, err)
    } else {
        log.Println("Saved:", filename)
    }
}

func main() {
    bytes, err := ioutil.ReadFile("celebrity_clean.json")
    if err != nil {
        log.Fatal(err)
    }
    var celebrities []Celebrity
    if err := json.Unmarshal(bytes, &celebrities); err != nil {
        log.Fatal(err)
    }
    var w sync.WaitGroup
    for idx, c := range celebrities {
        if idx&10 == 0 {
            w.Wait()
        }
        w.Add(1)
        go download(c.Image, c.Name+"_"+c.Yomi+".jpg", &w)
    }
    w.Wait()
}