Search Google - nodebotrpa/editor GitHub Wiki
This tutorial introduces NodeBot RPA and creates a flow that demonstrates search a keywork on google.com and writing to csv file.
How to Search keyword on Google and save result to file.
1. Add an inject node
2. Open a blank browser
Add a Web New Session node
3. Navigate to http://www.google.com
Add a Web Navigate to node. Enter http://www.google.com to URL field.
4. Search nodered
Add a Web Send Keys node. Enter //input[@name="q"] to element field and nodered{ENTER} to text field. How to inspect elements
5. Get result count
Add a Web Get Element node. Choose count action, Enter //*[@id="rso"]//div[@class="rc"] to element field, enable wait until visible property and resultCount to variable name field. Count of search result will return to msg.payload .
6. Initialize counters
Add function node.
flow.set('counter',1);
flow.set('max',parseInt(msg.payload));
return msg;
6. Generate title xpath
Add function node. There is an advertisement at first record.
msg.payload = '(//*[@id="rso"]//div[@class="rc"])['+flow.get('counter')+']//a[1]/h3';
return msg;
6. Get Title
Add a Web Get Element node. Choose Text action, enter msg.payload to element field and enter linkTitle to variable name field.
7. Generate url xpath
Add function node. There is an advertisement at first record.
msg.payload = '(//*[@id="rso"]//div[@class="rc"])['+flow.get('counter')+']//a[1]';
8. Get Url
Add a Web Get Element node. Choose Attirbute action, enter msg.payload to element field, enter href to attribute field and enter linkUrl to variable name field.
9. Read Title variable
Add Get Variable node. Enter linkTitle to variable field.
10. Generate output
Add function node
flow.set('result',msg.payload+';');
return msg;
11. Read Url variable
Add Get Variable node. Enter linkUrl to variable field.
12. Generate output
Add function node
flow.set('result',flow.get('result')+msg.payload);
return msg;
13. Set msg.payload
Add Change node. Set msg.payload to flow.result
14. Write to file
Add File Node. Enter C:\Temp\result.txt to filename field, choose append file action and enable add newline option.
15. Increase counter
Add Function node.
flow.set('counter',flow.get('counter')+1);
return msg;
16. Check flow counter for last record
Add Switch node. Enter flow.counter to property field. if <= flow.max go to step 6 and if > flow.max continue
17. Close browser
Add Window Action node and choose close action.
18. Handle Exception
Add Catch node to handle videos in search list. Choose selected nodes and select Get Title node. Link this node to increase counter, step 15.
Deploy and Run
Connect all nodes by dragging between nodes. Click Deploy button and Click inject node to run.
Import this flow
You can import this flow to NodeRed editor also.
- Copy below json data to clipboard
- Run NodeRed editor
- Click import menu
- Select Clipboard option and paste from clipboard.
- Click Import button
[{"id":"28b4398e.2eac76","type":"tab","label":"Search Google","disabled":false,"info":""},{"id":"7d439371.9723dc","type":"inject","z":"28b4398e.2eac76","name":"","repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":120,"y":80,"wires":["4e9ed68a.6e1fc8"](/nodebotrpa/editor/wiki/"4e9ed68a.6e1fc8")},{"id":"d0f62763.8db708","type":"debug","z":"28b4398e.2eac76","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":350,"y":560,"wires":[]},{"id":"4e9ed68a.6e1fc8","type":"nbr-web-new-session","z":"28b4398e.2eac76","name":"Open browser","browser":"CHROME","implicit":"0","script":"0","pageLoad":"0","maximize":false,"variable":"google","waitbefore":"500","waitafter":"500","x":300,"y":80,"wires":["2b94fc54.8c97f4"](/nodebotrpa/editor/wiki/"2b94fc54.8c97f4")},{"id":"2b94fc54.8c97f4","type":"nbr-web-navigate-to","z":"28b4398e.2eac76","name":"navigate to google.com","url":"http://www.google.com","pageLoad":false,"waitbefore":"500","waitafter":"500","x":530,"y":80,"wires":["b7fd3728.c2ddd8"](/nodebotrpa/editor/wiki/"b7fd3728.c2ddd8")},{"id":"b7fd3728.c2ddd8","type":"nbr-web-send-keys","z":"28b4398e.2eac76","name":"Search nodered","xpath":"//input[@name=\"q\"]","text":"nodered{ENTER}","clear":false,"waitvisible":false,"waitbefore":"500","waitafter":"500","x":860,"y":120,"wires":["76f88477.2a897c"](/nodebotrpa/editor/wiki/"76f88477.2a897c")},{"id":"c3d09e2e.553d7","type":"function","z":"28b4398e.2eac76","name":"generate xpath","func":"msg.payload = '(//*[@id=\"rso\"]//div[@class=\"rc\"])['+flow.get('counter')+']//a[1]/h3';\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":140,"y":220,"wires":["1922c2d1.764c1d"](/nodebotrpa/editor/wiki/"1922c2d1.764c1d")},{"id":"87856743.005f88","type":"function","z":"28b4398e.2eac76","name":"set counter to 1","func":"flow.set('counter',1);\nflow.set('max',parseInt(msg.payload));\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":340,"y":140,"wires":["c3d09e2e.553d7"](/nodebotrpa/editor/wiki/"c3d09e2e.553d7")},{"id":"1922c2d1.764c1d","type":"nbr-web-get-element","z":"28b4398e.2eac76","name":"Get Title","action":0,"waitvisible":false,"xpath":"msg.payload","attr":"","variable":"linkTitle","waitbefore":"500","waitafter":"500","x":320,"y":220,"wires":["f6c1f902.c46d28"](/nodebotrpa/editor/wiki/"f6c1f902.c46d28")},{"id":"f6c1f902.c46d28","type":"function","z":"28b4398e.2eac76","name":"generate xpath","func":"msg.payload = '(//*[@id=\"rso\"]//div[@class=\"rc\"])['+flow.get('counter')+']//a[1]';\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":500,"y":220,"wires":["3cfa5405.9165dc"](/nodebotrpa/editor/wiki/"3cfa5405.9165dc")},{"id":"3cfa5405.9165dc","type":"nbr-web-get-element","z":"28b4398e.2eac76","name":"Get Url","action":"1","waitvisible":false,"xpath":"msg.payload","attr":"href","variable":"linkUrl","waitbefore":"500","waitafter":"500","x":680,"y":220,"wires":["55ee32c5.1a70cc"](/nodebotrpa/editor/wiki/"55ee32c5.1a70cc")},{"id":"c390c2d4.fecec","type":"function","z":"28b4398e.2eac76","name":"Increase counter","func":"flow.set('counter',flow.get('counter')+1);\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":730,"y":380,"wires":["41246e04.f95db"](/nodebotrpa/editor/wiki/"41246e04.f95db")},{"id":"6a075812.58d448","type":"catch","z":"28b4398e.2eac76","name":"","scope":["1922c2d1.764c1d"],"uncaught":false,"x":830,"y":300,"wires":["c390c2d4.fecec"](/nodebotrpa/editor/wiki/"c390c2d4.fecec")},{"id":"76f88477.2a897c","type":"nbr-web-get-element","z":"28b4398e.2eac76","name":"Get result count","action":"7","waitvisible":true,"xpath":"//*[@id=\"rso\"]//div[@class=\"rc\"]","attr":"","variable":"resultCount","waitbefore":"500","waitafter":"500","x":140,"y":140,"wires":["87856743.005f88"](/nodebotrpa/editor/wiki/"87856743.005f88")},{"id":"41246e04.f95db","type":"switch","z":"28b4398e.2eac76","name":"","property":"counter","propertyType":"flow","rules":[{"t":"lte","v":"max","vt":"flow"},{"t":"gt","v":"max","vt":"flow"}],"checkall":"true","repair":false,"outputs":2,"x":110,"y":500,"wires":["c3d09e2e.553d7"],["1005ed0.b337c13"](/nodebotrpa/editor/wiki/"c3d09e2e.553d7"],["1005ed0.b337c13")},{"id":"1005ed0.b337c13","type":"nbr-web-window-action","z":"28b4398e.2eac76","name":"Close browser","action":7,"waitbefore":"500","waitafter":"500","x":320,"y":500,"wires":["d0f62763.8db708"](/nodebotrpa/editor/wiki/"d0f62763.8db708")},{"id":"55ee32c5.1a70cc","type":"nbr-get-variable","z":"28b4398e.2eac76","name":"","variable":"linkTitle","waitbefore":"500","waitafter":"500","x":170,"y":300,"wires":["f28f8ecb.46704"](/nodebotrpa/editor/wiki/"f28f8ecb.46704")},{"id":"f28f8ecb.46704","type":"function","z":"28b4398e.2eac76","name":"store title","func":"flow.set('result',msg.payload+';');\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":320,"y":300,"wires":["33e6e005.1e071"](/nodebotrpa/editor/wiki/"33e6e005.1e071")},{"id":"33e6e005.1e071","type":"nbr-get-variable","z":"28b4398e.2eac76","name":"","variable":"linkUrl","waitbefore":"500","waitafter":"500","x":470,"y":300,"wires":["4d0bab2e.639cd4"](/nodebotrpa/editor/wiki/"4d0bab2e.639cd4")},{"id":"4d0bab2e.639cd4","type":"function","z":"28b4398e.2eac76","name":"Store url","func":"flow.set('result',flow.get('result')+msg.payload);\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":640,"y":300,"wires":["4864efea.caeac"](/nodebotrpa/editor/wiki/"4864efea.caeac")},{"id":"9430e4e7.215698","type":"file","z":"28b4398e.2eac76","name":"","filename":"C:\\Temp\\result.txt","appendNewline":true,"createDir":false,"overwriteFile":"false","encoding":"none","x":490,"y":380,"wires":["c390c2d4.fecec"](/nodebotrpa/editor/wiki/"c390c2d4.fecec")},{"id":"4864efea.caeac","type":"change","z":"28b4398e.2eac76","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"result","tot":"flow"}],"action":"","property":"","from":"","to":"","reg":false,"x":280,"y":380,"wires":["9430e4e7.215698"](/nodebotrpa/editor/wiki/"9430e4e7.215698")}]