网络爬虫抓取图片并上传到fastdfs图片不完整问题解决网络爬虫抓取图片并上传到f

注意：如果直接使用uri的连接将图片上传到fastdfs会导致图片上传不完整，所以需要将图片先保存下来，上传完成后再删除
Imagedownload

public static String fastdfsdownload(String imgUrl) { String filepath = ""; File file = null; try { // 获取图片URL URL url = new URL(imgUrl); // 获得连接 URLConnection connection = url.openConnection(); // 设置10秒的相应时间 connection.setConnectTimeout(10 * 1000); //模拟浏览器访问，防止网站屏蔽 connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.19 Safari/537.36"); InputStream is = connection.getInputStream(); if (is == null || is.available() <= 0) { return null; } //得到临时文件 file = getTemplateFile(is); filepath = fastDFS.upload(file); is.close(); } catch (IOException e) { e.printStackTrace(); } return filepath; }public static File getTemplateFile(InputStream inputStream) { File file = null; try { file = File.createTempFile("temp_image", null); } catch (IOException e) { e.printStackTrace(); } inputStreamToFile(inputStream, file); if (file.exists() && file.length() <= 0) { return null; } return file; }public static void inputStreamToFile(InputStream ins, File file) { try { OutputStream os = new FileOutputStream(file); try { int bytesRead = 0; byte[] buffer = new byte[8192]; while ((bytesRead = ins.read(buffer, 0, 8192)) != -1) { os.write(buffer, 0, bytesRead); } os.flush(); } finally { if (os != null) { os.close(); } if (ins != null) { ins.close(); } } } catch (Exception e) { e.printStackTrace(); } }

【网络爬虫抓取图片并上传到fastdfs图片不完整问题解决】FastDFS

package com.neusoft.util; import com.neusoft.fastdfsTemlate.FastDFSTemplate; import com.neusoft.fastdfsTemlate.FastDfsInfo; import com.neusoft.fastdfsTemlate.exception.FastDFSException; import org.csource.common.NameValuePair; import org.springframework.beans.factory.BeanFactory; import org.springframework.context.support.ClassPathXmlApplicationContext; import java.io.*; import java.util.HashMap; import java.util.Map; public class FastDFS { private BeanFactory beanFactory = new ClassPathXmlApplicationContext("classpath:spring-fastdfs.xml"); private FastDFSTemplate dfsTemplate = (FastDFSTemplate) beanFactory.getBean("fastDFSTemplate"); public String upload(File tempFile) { InputStream is = null; try { is = new FileInputStream(tempFile); } catch (FileNotFoundException e) { e.printStackTrace(); } byte[] bytes = null; try { bytes = new byte[is.available()]; int length = is.read(bytes); } catch (IOException e) { e.printStackTrace(); } Map map = new HashMap<>(); FastDfsInfo rv = null; try { rv = dfsTemplate.upload(bytes, "jpg", map); } catch (FastDFSException e) { e.printStackTrace(); }finally { if (is != null) { try { is.close(); } catch (IOException e) { e.printStackTrace(); } } if (tempFile != null) { tempFile.delete(); } } assert rv != null; return rv.getFileAbsolutePath(); } }