Java中常见的编码集问题总结

package com.guor.demo.charset; import java.io.BufferedReader; import java.io.FileReader; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; public class CommonUtils { public static List> readCSVToList(String filePath) throws Exception { List> list = new ArrayList>(); BufferedReader reader = null; try { reader = new BufferedReader(new FileReader(filePath)); String[] headtilte = reader.readLine().split(","); String line = null; while ((line = reader.readLine()) != null) { HashMap hashMap = new HashMap(); String[] itemArray = line.split(","); for (int i = 0; i 

public static List readCSVToListByUnicode(String filePath) { FileInputStream fis = null; UnicodeInputStream uin = null; InputStreamReader in= null; try { //logger.info("读取文件"+filePath+"编码集"); List list = new ArrayList(); //URL url = new URL(filePath); File f  = new File(filePath); fis = new FileInputStream(f); uin = new UnicodeInputStream(fis,enc);  //如果是本地将url.openStream -> new FileInputStream(f) enc = uin.getEncoding(); // check and skip possible BOM bytes if (enc == null){ in = new InputStreamReader(uin); }else { in = new InputStreamReader(uin, enc); } long start = System.currentTimeMillis(); //logger.info("读取文件"+filePath+"查看耗时开始"); BufferedReader reader = new BufferedReader(in); //BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream("D:/tags.txt"),"utf-8")); //String tmp =reader.readLine(); String[] headtilte = reader.readLine().split(","); list.add(headtilte); String line = null; String[] itemArray = null; while ((line = reader.readLine()) != null) { itemArray = line.split(","); list.add(itemArray); } long end = System.currentTimeMillis(); //logger.info("读取文件"+filePath+"查看耗时="+(end-start)+"毫秒"); return list; }catch (Exception e){ logger.info("读取文件"+filePath+"，异常：", e); return null; }finally { try { if(in!=null){ in.close(); } if(uin!=null){ uin.close(); } if(fis!=null){ fis.close(); } }catch (Exception e){ logger.info("读取文件"+filePath+"，关闭流异常：", e); } } }

package com.guor.demo.utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.io.PushbackInputStream; public class UnicodeInputStream extends InputStream { private static final Logger logger = LoggerFactory.getLogger(UnicodeInputStream.class); PushbackInputStream internalIn; boolean isInited = false; String defaultEnc; String encoding; private static final int BOM_SIZE = 4; public UnicodeInputStream(InputStream in, String defaultEnc) { internalIn = new PushbackInputStream(in, BOM_SIZE); this.defaultEnc = defaultEnc; } public String getDefaultEncoding() { return defaultEnc; } public String getEncoding() { if (!isInited) { try { init(); } catch (IOException ex) { IllegalStateException ise = new IllegalStateException("Init method failed."); ise.initCause(ise); throw ise; } } return encoding; } /** * Read-ahead four bytes and check for BOM marks. Extra bytes are * unread back to the stream, only BOM bytes are skipped. */ protected void init() throws IOException { if (isInited) { return; } byte bom[] = new byte[BOM_SIZE]; int n, unread; n = internalIn.read(bom, 0, bom.length); //logger.info("文件的前四个字符==="+bom[0]+","+bom[1]+","+bom[2]+","+bom[3]); if ( (bom[0] == (byte)0x00) && (bom[1] == (byte)0x00) && (bom[2] == (byte)0xFE) && (bom[3] == (byte)0xFF) ) { encoding = "UTF-32BE"; unread = n - 4; } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) && (bom[2] == (byte)0x00) && (bom[3] == (byte)0x00) ) { encoding = "UTF-32LE"; unread = n - 4; } else if (  (bom[0] == (byte)0xEF) && (bom[1] == (byte)0xBB) && (bom[2] == (byte)0xBF) ) { encoding = "UTF-8"; unread = n - 3; } else if ( (bom[0] == (byte)0xFE) && (bom[1] == (byte)0xFF) ) { encoding = "UTF-16BE"; unread = n - 2; } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) ) { encoding = "UTF-16LE"; unread = n - 2; } else { // Unicode BOM mark not found, unread all bytes encoding = defaultEnc; unread = n; } //System.out.println("read=" + n + ", unread=" + unread); if (unread > 0) { internalIn.unread(bom, (n - unread), unread); } isInited = true; } @Override public void close() throws IOException { //init(); isInited = true; internalIn.close(); } @Override public int read() throws IOException { //init(); isInited = true; return internalIn.read(); } }