Last active
October 1, 2015 23:18
-
-
Save repeatedly/2114104 to your computer and use it in GitHub Desktop.
D言語でいかにしておっぱい画像をダウンロードするか〜2013
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // Written in the D programming language. | |
| /** | |
| * High peformance downloader | |
| * | |
| * Implemented according to <a href="http://yusukebe.com/archives/20120229/072808.html">this implementation</a>. | |
| * | |
| * Example: | |
| * ----- | |
| * dmd -L-lcurl -run downloader.d | |
| * ----- | |
| */ | |
| import core.atomic : atomicOp; | |
| import std.array : join; | |
| import std.conv : to; | |
| import std.digest.md : md5Of, toHexString; | |
| import std.getopt : getopt; | |
| import std.file : exists, mkdir; | |
| import std.json : parseJSON, toJSON; | |
| import std.path : buildPath, setExtension; | |
| import std.parallelism : parallel; | |
| import std.net.curl : get, download; | |
| import std.regex : match; | |
| import std.string : toLower; | |
| import std.stdio : writeln; | |
| import std.uri : encode; | |
| @trusted | |
| string buildQuery(in string[string] params) | |
| { | |
| string[] queries; | |
| foreach (k, v; params) | |
| queries ~= encode(k) ~ "=" ~ encode(v); | |
| return queries.join("&"); | |
| } | |
| void main(string[] args) | |
| { | |
| immutable AppURI = "http://api.bing.net/json.aspx"; | |
| immutable AppId = "Your AppId"; | |
| string storeDir = "./op_images"; | |
| string query = "おっぱい"; | |
| size_t page = 0; | |
| getopt(args, | |
| "query", &query, | |
| "dir", &storeDir, | |
| "page", &page); | |
| if (!storeDir.exists) | |
| mkdir(storeDir); | |
| writeln("Configuration: query = ", query, ", dir = ", storeDir, ", offset = ", page); | |
| for (shared(size_t) downloadCount; true; page++) { | |
| immutable uri = AppURI ~ '?' ~ buildQuery(["AppId": AppId, | |
| "Version": "2.2", | |
| "Market": "ja-JP", | |
| "Sources": "Image", | |
| "Image.Count": "50", | |
| "Image.Offset": to!string(page * 50), | |
| "Adult": "off", | |
| "Query": query]); | |
| auto response = get(uri); | |
| auto result = parseJSON(response).object["SearchResponse"]; | |
| if ("Errors" in result.object) { | |
| writeln("Error: reason = ", toJSON(&result.object["Errors"])); | |
| break; | |
| } | |
| foreach (ref entry; parallel(result.object["Image"].object["Results"].array)) { | |
| immutable downloadUri = entry.object["MediaUrl"].str; | |
| if (!downloadUri.match(".jpe?g$")) | |
| continue; | |
| immutable path = buildPath(storeDir, md5Of(downloadUri).toHexString.toLower.setExtension("jpg")); | |
| if (path.exists) | |
| continue; | |
| download(downloadUri, path); | |
| atomicOp!"+="(downloadCount, 1); | |
| writeln(downloadCount, " : downloaded... ", downloadUri); | |
| } | |
| } | |
| writeln("done!"); | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment