本文方法及代碼僅供學習，僅供學習。
案例：
下載酷狗TOP500歌曲，代碼用到的代碼庫包含：Jsoup、HttpClient、fastJson等。
正文：
1、分析是否可以獲取到TOP500歌單
打開酷狗首頁，查看TOP500，發現存在分頁，每頁顯示22條歌曲，
發現酷狗的鏈接如下：
https://www.kugou.com/yy/rank/home/1-8888.html?from=homepage
通過更改鏈接中的1可以進行分頁，所以我們可以通過更改鏈接地址獲取其余的歌曲。
2、分析找到正真的mp3下載地址
?點一個歌曲進入播放頁面，使用谷歌瀏覽器的控制臺的Elements，搜一下mp3，很輕松就定位到了MP3的位置。
但是使用java訪問的時候爬取的html里卻沒有該mp3的文件地址，那么這肯定是在該頁面的位置使用了js來加載mp3，那么刷新下網頁，看網頁加載了哪些東西，加載的東西有點多，著重看一下js、php的請求，主要是看里面有沒有mp3的地址。
最終在列表中找到：
https://wwwapi.kugou.com/yy/index.php?r=play/getdata&callback=jQuery191044492686523157987_1559446927765&hash=458E9B9F362277AC37E9EEF1CB80B535&album_id=18712576&dfid=1ZxQbe0MiP8J09j5tR0Np9IA&mid=9393340fecff864a4d6c4e95099b2be1&platid=4&_=1559446927766
這個請求結果中發現了mp3的完整地址：
那這個js是怎么判斷是哪首歌的呢，那么只可能是hash這個參數來決定歌曲的，然后到播放頁面里找到這個hash的位置，是在下面的js里：
var dataFromSmarty = [{"hash":"667939C6E784265D541DEEE65AE4F2F8","timelength":"237051","audio_name":"\u767d\u5c0f\u767d - \u6700\u7f8e\u5a5a\u793c","author_name":"\u767d\u5c0f\u767d","song_name":"\u6700\u7f8e\u5a5a\u793c","album_id":0}],//當前頁面歌曲信息playType = "search_single";//當前播放</script>
在去java爬取該網頁，查看能否爬到這個hash，果然，爬取的html里有這段js，到現在mp3的地址也找到了，歌單也找到了，那么下一步就用程序實現就可以了。
3、代碼實現
SpiderKugou.java
package com.billy.test;import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements;import java.io.IOException; import java.util.regex.Matcher; import java.util.regex.Pattern;/*** 爬取并下載酷狗的歌曲*/ public class SpiderKugou {private static String filePath;//酷狗地址private static String LINK;//mp3地址private static String mp3;static {filePath = "F:/music/";LINK = "https://www.kugou.com/yy/rank/home/PAGE-8888.html?from=rank";mp3 = "https://wwwapi.kugou.com/yy/index.php?r=play/getdata&callback=jQuery19103632090130122796_1558800325111&"+ "hash=HASH&_=TIME";}public static void main(String[] args) throws Exception {for(int i = 5 ; i < 23 ; i++){String url = LINK.replace("PAGE", i + "");downSong(url);}}/*** 下載歌曲* @param url* @throws Exception*/private static void downSong(String url) throws Exception{HttpGetConnect connect = new HttpGetConnect();String content = connect.connect(url, "utf-8");HtmlManage html = new HtmlManage();Document doc = html.manage(content);Element ele = doc.getElementsByClass("pc_temp_songlist").get(0);Elements elements = ele.getElementsByTag("li");for(int i = 0 ; i < elements.size() ; i++){Element item = elements.get(i);String title = item.attr("title").trim();String link = item.getElementsByTag("a").first().attr("href");downLoad(link,title);Thread.sleep(1000);}}/*** 下載* @param url* @param name* @throws IOException*/private static void downLoad(String url,String name) throws IOException{String hash = "";HttpGetConnect connect = new HttpGetConnect();String content = connect.connect(url, "utf-8");String regEx = "\"hash\":\"[0-9A-Z]+\"";// 編譯正則表達式Pattern pattern = Pattern.compile(regEx);Matcher matcher = pattern.matcher(content);if (matcher.find()) {hash = matcher.group();hash = hash.replace("\"hash\":\"", "");hash = hash.replace("\"", "");}String item = mp3.replace("HASH", hash);item = item.replace("TIME", System.currentTimeMillis() + "");System.out.println("item:" + item);String mp = connect.connect(item, "utf-8");mp = mp.substring(mp.indexOf("(") + 1, mp.length() - 3);JSONObject json = JSON.parseObject(mp);if(Integer.parseInt(json.get("status") + "") != 0){String playUrl = json.getJSONObject("data").getString("play_url");FileDownload down = new FileDownload();down.download(playUrl, filePath + name + ".mp3");}}}
HttpGetConnect.java
package com.billy.test;import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpEntity; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.BasicHttpClientConnectionManager;import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader;/*** httpclient 工具類*/ public class HttpGetConnect {/*** 獲取html內容* @param url* @param charsetName UTF-8、GB2312* @return* @throws IOException*/public static String connect(String url,String charsetName) throws IOException{BasicHttpClientConnectionManager connManager = new BasicHttpClientConnectionManager();CloseableHttpClient httpclient = HttpClients.custom().setConnectionManager(connManager).build();String content = "";try{HttpGet httpget = new HttpGet(url);RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(5000).setConnectTimeout(50000).setConnectionRequestTimeout(50000).build();httpget.setConfig(requestConfig);httpget.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");httpget.setHeader("Accept-Encoding", "gzip,deflate,sdch");httpget.setHeader("Accept-Language", "zh-CN,zh;q=0.8");httpget.setHeader("Connection", "keep-alive");httpget.setHeader("Upgrade-Insecure-Requests", "1");httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");httpget.setHeader("cache-control", "max-age=0");httpget.setHeader("Referer","https://www.kugou.com/song/");//設置cookiehttpget.setHeader("Cookie", "kg_mid=9393340fecff864a4d6c4e95099b2be1;");CloseableHttpResponse response = httpclient.execute(httpget);int status = response.getStatusLine().getStatusCode();if (status >= 200 && status < 300) {HttpEntity entity = response.getEntity();InputStream instream = entity.getContent();BufferedReader br = new BufferedReader(new InputStreamReader(instream,charsetName));StringBuffer sbf = new StringBuffer();String line = null;while ((line = br.readLine()) != null){sbf.append(line + "\n");}br.close();content = sbf.toString();} else {content = "";}}catch(Exception e){e.printStackTrace();}finally{httpclient.close();}log.info("content is " + content);return content;}private static Log log = LogFactory.getLog(HttpGetConnect.class); }
HtmlManage.java
package com.billy.test;import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements;import java.io.IOException; import java.util.ArrayList; import java.util.List;/*** html manage 工具類*/ public class HtmlManage {public Document manage(String html) {Document doc = Jsoup.parse(html);return doc;}public Document manageDirect(String url) throws IOException {Document doc = Jsoup.connect(url).get();return doc;}public List<String> manageHtmlTag(Document doc, String tag) {List<String> list = new ArrayList<String>();Elements elements = doc.getElementsByTag(tag);for (int i = 0; i < elements.size(); i++) {String str = elements.get(i).html();list.add(str);}return list;}public List<String> manageHtmlClass(Document doc, String clas) {List<String> list = new ArrayList<String>();Elements elements = doc.getElementsByClass(clas);for (int i = 0; i < elements.size(); i++) {String str = elements.get(i).html();list.add(str);}return list;}public List<String> manageHtmlKey(Document doc, String key, String value) {List<String> list = new ArrayList<String>();Elements elements = doc.getElementsByAttributeValue(key, value);for (int i = 0; i < elements.size(); i++) {String str = elements.get(i).html();list.add(str);}return list;}private static Log log = LogFactory.getLog(HtmlManage.class); }
FileDownload.java
package com.billy.test;import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream;import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients;/*** 文件下載工具類*/ public class FileDownload {/*** 文件下載** @param url 鏈接地址* @param path 要保存的路徑及文件名* @return*/public static boolean download(String url, String path) {boolean flag = false;CloseableHttpClient httpclient = HttpClients.createDefault();RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(2000).setConnectTimeout(2000).build();HttpGet get = new HttpGet(url);get.setConfig(requestConfig);BufferedInputStream in = null;BufferedOutputStream out = null;try {for (int i = 0; i < 3; i++) {CloseableHttpResponse result = httpclient.execute(get);System.out.println(result.getStatusLine());if (result.getStatusLine().getStatusCode() == 200) {in = new BufferedInputStream(result.getEntity().getContent());File file = new File(path);out = new BufferedOutputStream(new FileOutputStream(file));byte[] buffer = new byte[1024];int len = -1;while ((len = in.read(buffer, 0, 1024)) > -1) {out.write(buffer, 0, len);}flag = true;break;} else if (result.getStatusLine().getStatusCode() == 500) {continue;}}} catch (Exception e) {e.printStackTrace();flag = false;} finally {get.releaseConnection();try {if (in != null) {in.close();}if (out != null) {out.close();}} catch (Exception e) {e.printStackTrace();flag = false;}}return flag;}private static Log log = LogFactory.getLog(FileDownload.class); }
?
?
?
?
?
?
: “TypeError: Cannot read properties of undefined (reading ‘graphic‘)“)
