// Copyright (C) 2014 Adriano Soares
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see .
package main
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"strings"
)
type WikiaResult map[string]struct {
Title string
Revisions []struct {
Text string `json:"*"`
}
}
var resultJSON = make(WikiaResult)
var missedCards []string
func wikia() {
cards := getListOfPages()
// download
const step = 50
var size = len(cards)
for i := 0; i < size; i += step {
fmt.Println(i, "of", size)
if i+step > size {
parseJSON(cards[i:])
} else {
parseJSON(cards[i : i+step])
}
}
save(resultJSON, config.Db)
}
func parseJSON(ids []string) {
resp, err := http.PostForm(config.Api,
url.Values{
"action": {"query"},
"format": {"json"},
"redirects": {"1"},
"prop": {"revisions"},
"rvprop": {"content"},
"titles": {strings.Join(ids, "|")},
})
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
var rawData struct {
Query struct{ Pages json.RawMessage }
}
err = json.NewDecoder(resp.Body).Decode(&rawData)
catch(err)
var cards WikiaResult
err = json.Unmarshal(rawData.Query.Pages, &cards)
catch(err)
for id, c := range cards {
resultJSON[id] = c
}
}