1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
| "github.com/anaskhan96/soup"
"fmt"
"io/ioutil"
"net/http"
"os"
"runtime"
doc := getDoc(url)
func getDoc(url string) soup.Root {
req, err := http.NewRequest("GET", url, nil)
printError(err)
res, err := http.DefaultClient.Do(req)
printError(err)
body, err := ioutil.ReadAll(res.Body)
printError(err)
resp := string(body)
//save(resp, "/tmp/log.html")
return soup.HTMLParse(resp)
}
/*func getDoc(url string) soup.Root {
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{
Timeout: 15 * time.Second,
Transport: tr,
}
req, err := http.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0")
printError(err)
resp, err := client.Do(req)
printError(err)
body, err := ioutil.ReadAll(resp.Body)
printError(err)
return soup.HTMLParse(string(body))
}*/
"github.com/sclevine/agouti"
func openBrowser(screenX, screenY int) (*agouti.Page, *agouti.WebDriver) {
driver := agouti.ChromeDriver(
agouti.ChromeOptions("args", []string{
"--headless",
"--disable-gpu",
"--disable-notifications",
//"--no-sandbox",
fmt.Sprintf("--window-size=%v,%v", screenX, screenY),
}),
)
err := driver.Start()
printError(err)
page, err := driver.NewPage()
printError(err)
return page, driver
}
func main (){
page, driver := openBrowser()
defer driver.Stop()
page.Navigate(url)
resp, _ := page.HTML()
doc := soup.HTMLParse(resp)
}
func runJs(page *agouti.Page) string{
js := `window.scrollTo(0,window.scrollY+window.innerHeight*.9);`
var text string
page.RunScript(js, nil, &text)
return text
}
func sendKeys(page *agouti.Page, key string) {
var k string
switch key {
case "Home":
k = "\uE011"
case "pageDown":
k = "\uE00F"
}
page.Find("html").SendKeys(k)
// NULL = "\uE000", CANCEL = "\uE001", HELP = "\uE002"
// BACK_SPACE = "\uE003", TAB = "\uE004", CLEAR = "\uE005"
// RETURN = "\uE006", ENTER = "\uE007", SHIFT = "\uE008"
// CONTROL = "\uE009", ALT = "\uE00A", PAUSE = "\uE00B"
// ESCAPE = "\uE00C", SPACE = "\uE00D", PAGE_UP = "\uE00E"
// PAGE_DOWN = "\uE00F", END = "\uE010", HOME = "\uE011"
// LEFT = "\uE012", ARROW_LEFT = "\uE012", ARROW_UP = "\uE013"
// ARROW_RIGHT = "\uE014", ARROW_DOWN = "\uE015", INSERT = "\uE016"
// DELETE = "\uE017", SEMICOLON = "\uE018", EQUALS = "\uE019"
// NUMPAD0 = "\uE01A", NUMPAD1 = "\uE01B", NUMPAD2 = "\uE01C"
// NUMPAD3 = "\uE01D", NUMPAD4 = "\uE01E", NUMPAD5 = "\uE01F"
// NUMPAD6 = "\uE020", NUMPAD7 = "\uE021", NUMPAD8 = "\uE022"
// NUMPAD9 = "\uE023", MULTIPLY = "\uE024", ADD = "\uE025"
// SEPARATOR = "\uE026", SUBTRACT = "\uE027", DECIMAL = "\uE028"
// DIVIDE = "\uE029", F1 = "\uE031", F2 = "\uE032"
// F3 = "\uE033", F4 = "\uE034", F5 = "\uE035"
// F6 = "\uE036", F7 = "\uE037", F8 = "\uE038"
// F9 = "\uE039", F10 = "\uE03A", F11 = "\uE03B"
// F12 = "\uE03C", META = "\uE03D", COMMAND = "\uE03D"
}
|