org.dice_research.squirrel.analyzer.impl.html.scraper

Class HtmlScraper

java.lang.Object

org.dice_research.squirrel.analyzer.impl.html.scraper.HtmlScraper

public class HtmlScraper
extends Object

HTMLScraper to extract triples from HTML Data based in pre configured yaml files.

Author:

gsjunior

Field Summary

Fields

Modifier and Type	Field and Description
private org.jsoup.nodes.Document	doc
private String	label
private LinkedHashSet<String>	listIterableObjects
private static org.slf4j.Logger	LOGGER
private Map<String,List<org.apache.jena.graph.Triple>>	selectedMap
private Map<String,List<org.apache.jena.graph.Triple>>	staticMap
static Comparator<org.apache.jena.graph.Triple>	tripleComparator
private LinkedHashSet<org.apache.jena.graph.Node>	updatedObjects
private String	uri
private Map<String,YamlFile>	yamlFiles

Constructor Summary

Constructors

Constructor and Description
HtmlScraper()
HtmlScraper(File file)

Method Summary

Modifier and Type	Method and Description
private List<org.apache.jena.graph.Node>	jsoupQuery(String cssQ)
private String	replaceCommands(String s)
List<org.apache.jena.graph.Triple>	scrape(String uri, File filetToScrape)
private Set<org.apache.jena.graph.Triple>	scrapeDownloadLink(Map<String,Object> resources, File htmlFile)
private Set<org.apache.jena.graph.Triple>	scrapeTree(Map<String,Object> mapEntry, Set<org.apache.jena.graph.Triple> triples, Stack<org.apache.jena.graph.Node> stackNode)
private List<org.apache.jena.graph.Triple>	updateRelationship(List<org.apache.jena.graph.Triple> listTriples) Update the triples with nested objects

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOGGER

private static final org.slf4j.Logger LOGGER

yamlFiles

private Map<String,YamlFile> yamlFiles

updatedObjects

private LinkedHashSet<org.apache.jena.graph.Node> updatedObjects

listIterableObjects

private LinkedHashSet<String> listIterableObjects

uri

private String uri

label

private String label

doc

private org.jsoup.nodes.Document doc

staticMap

private Map<String,List<org.apache.jena.graph.Triple>> staticMap

selectedMap

private Map<String,List<org.apache.jena.graph.Triple>> selectedMap

tripleComparator

public static Comparator<org.apache.jena.graph.Triple> tripleComparator

Constructor Detail

HtmlScraper

public HtmlScraper(File file)

HtmlScraper

public HtmlScraper()

Method Detail

scrape

public List<org.apache.jena.graph.Triple> scrape(String uri,
                                                 File filetToScrape)
                                          throws Exception

Throws:

Exception

updateRelationship

private List<org.apache.jena.graph.Triple> updateRelationship(List<org.apache.jena.graph.Triple> listTriples)

Update the triples with nested objects

Parameters:

listTriples -

Returns:

scrapeDownloadLink

private Set<org.apache.jena.graph.Triple> scrapeDownloadLink(Map<String,Object> resources,
                                                             File htmlFile)
                                                      throws Exception

Throws:

Exception

replaceCommands

private String replaceCommands(String s)

scrapeTree

private Set<org.apache.jena.graph.Triple> scrapeTree(Map<String,Object> mapEntry,
                                                     Set<org.apache.jena.graph.Triple> triples,
                                                     Stack<org.apache.jena.graph.Node> stackNode)
                                              throws Exception

Parameters:

mapEntry -

triples -

stackNode -

Returns:

Throws:

MalformedURLException

Exception

jsoupQuery

private List<org.apache.jena.graph.Node> jsoupQuery(String cssQ)
                                             throws Exception

Throws:

Exception