diff --git a/CHANGELOG.md b/CHANGELOG.md index 28f65152c..aee17cb55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ * 【core 】 LambdaUtil添加getFieldName（issue#I4750U@Gitee） * 【cron 】 Scheduler增加setThreadExecutor（issue#I47A6N@Gitee） * 【core 】 CharsetDetector增加detect重载，支持自定义缓存大小（issue#I478E5@Gitee） +* 【core 】 增加PartitionIter（pr#402@Gitee） ### 🐞Bug修复 * 【core 】 修复MapUtil.sort比较器不一致返回原map的问题（issue#I46AQJ@Gitee） diff --git a/hutool-core/src/main/java/cn/hutool/core/collection/Partition.java b/hutool-core/src/main/java/cn/hutool/core/collection/Partition.java index 861e7e0a6..40015104d 100644 --- a/hutool-core/src/main/java/cn/hutool/core/collection/Partition.java +++ b/hutool-core/src/main/java/cn/hutool/core/collection/Partition.java @@ -15,8 +15,8 @@ import java.util.List; */ public class Partition extends AbstractList> { - final List list; - final int size; + protected final List list; + protected final int size; /** * 列表分区 diff --git a/hutool-core/src/main/java/cn/hutool/core/collection/PartitionIter.java b/hutool-core/src/main/java/cn/hutool/core/collection/PartitionIter.java new file mode 100644 index 000000000..37b82c75d --- /dev/null +++ b/hutool-core/src/main/java/cn/hutool/core/collection/PartitionIter.java @@ -0,0 +1,64 @@ +package cn.hutool.core.collection; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * 分批迭代工具，可以分批处理数据 + *
    + *
  1. 比如调用其他客户的接口，传入的入参有限，需要分批
    2. + *
  2. 比如mysql/oracle用in语句查询，超过1000可以分批
    3. + *
  3. 比如数据库取出游标，可以把游标里的数据一批一批处理
    4. + *
+ * + * @param 字段类型 + * @author qiqi.chen + * @since 5.7.10 + */ +public class PartitionIter implements Iterator>, Iterable>, Serializable { + private static final long serialVersionUID = 1L; + + /** + * 被分批的迭代器 + */ + protected final Iterator iterator; + /** + * 实际每批大小 + */ + protected final int partitionSize; + + /** + * 创建分组对象 + * + * @param iterator 迭代器 + * @param partitionSize 每批大小，最后一批不满一批算一批 + */ + public PartitionIter(Iterator iterator, int partitionSize) { + this.iterator = iterator; + this.partitionSize = partitionSize; + } + + @Override + public Iterator> iterator() { + return this; + } + + @Override + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Override + public List next() { + final List list = new ArrayList<>(this.partitionSize); + for (int i = 0; i < this.partitionSize; i++) { + if (false == iterator.hasNext()) { + break; + } + list.add(iterator.next()); + } + return list; + } +} diff --git a/hutool-core/src/main/java/cn/hutool/core/io/FileUtil.java b/hutool-core/src/main/java/cn/hutool/core/io/FileUtil.java index 5437041dc..9b9f7f55f 100644 --- a/hutool-core/src/main/java/cn/hutool/core/io/FileUtil.java +++ b/hutool-core/src/main/java/cn/hutool/core/io/FileUtil.java @@ -685,8 +685,8 @@ public class FileUtil extends PathUtil { * 某个文件删除失败会终止删除操作 * *

- * 从5.7.6开始，删除文件使用{@link Files#delete(Path)}代替 {@link File#delete()}
- * 因为前者遇到文件被占用等原因时，抛出异常，而非返回false，异常会指明具体的失败原因。 + * 从5.7.6开始，删除文件使用{@link Files#delete(Path)}代替 {@link File#delete()}
+ * 因为前者遇到文件被占用等原因时，抛出异常，而非返回false，异常会指明具体的失败原因。 *

* * @param file 文件对象 @@ -712,7 +712,7 @@ public class FileUtil extends PathUtil { final Path path = file.toPath(); try { delFile(path); - } catch (DirectoryNotEmptyException e){ + } catch (DirectoryNotEmptyException e) { // 遍历清空目录没有成功，此时补充删除一次（可能存在部分软链） del(path); } catch (IOException e) { @@ -754,7 +754,7 @@ public class FileUtil extends PathUtil { final File[] files = directory.listFiles(); if (null != files) { for (File childFile : files) { - if(false == del(childFile)){ + if (false == del(childFile)) { // 删除一个出错则本次删除任务失败 return false; } @@ -1401,7 +1401,7 @@ public class FileUtil extends PathUtil { */ @Deprecated public static boolean isModifed(File file, long lastModifyTime) { - return isModified(file,lastModifyTime); + return isModified(file, lastModifyTime); } @@ -1815,9 +1815,11 @@ public class FileUtil extends PathUtil { * @param charsetName 字符集 * @return BufferedReader对象 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #getReader(File, Charset)} */ + @Deprecated public static BufferedReader getReader(File file, String charsetName) throws IORuntimeException { - return IoUtil.getReader(getInputStream(file), charsetName); + return IoUtil.getReader(getInputStream(file), CharsetUtil.charset(charsetName)); } /** @@ -1839,9 +1841,11 @@ public class FileUtil extends PathUtil { * @param charsetName 字符集 * @return BufferedReader对象 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #getReader(String, Charset)} */ + @Deprecated public static BufferedReader getReader(String path, String charsetName) throws IORuntimeException { - return getReader(file(path), charsetName); + return getReader(path, CharsetUtil.charset(charsetName)); } /** @@ -1912,7 +1916,9 @@ public class FileUtil extends PathUtil { * @param charsetName 字符集 * @return 内容 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #readString(File, Charset)} */ + @Deprecated public static String readString(File file, String charsetName) throws IORuntimeException { return readString(file, CharsetUtil.charset(charsetName)); } @@ -1936,9 +1942,11 @@ public class FileUtil extends PathUtil { * @param charsetName 字符集 * @return 内容 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #readString(String, Charset)} */ + @Deprecated public static String readString(String path, String charsetName) throws IORuntimeException { - return readString(file(path), charsetName); + return readString(path, CharsetUtil.charset(charsetName)); } /** @@ -1953,6 +1961,20 @@ public class FileUtil extends PathUtil { return readString(file(path), charset); } + /** + * 读取文件内容 + * + * @param url 文件URL + * @param charsetName 字符集 + * @return 内容 + * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #readString(URL, Charset)} + */ + @Deprecated + public static String readString(URL url, String charsetName) throws IORuntimeException { + return readString(url, CharsetUtil.charset(charsetName)); + } + /** * 读取文件内容 * @@ -1960,8 +1982,9 @@ public class FileUtil extends PathUtil { * @param charset 字符集 * @return 内容 * @throws IORuntimeException IO异常 + * @since 5.7.10 */ - public static String readString(URL url, String charset) throws IORuntimeException { + public static String readString(URL url, Charset charset) throws IORuntimeException { if (url == null) { throw new NullPointerException("Empty url provided!"); } @@ -2083,7 +2106,9 @@ public class FileUtil extends PathUtil { * @param collection 集合 * @return 文件中的每行内容的集合 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #readLines(URL, Charset, Collection)} */ + @Deprecated public static > T readLines(URL url, String charsetName, T collection) throws IORuntimeException { return readLines(url, CharsetUtil.charset(charsetName), collection); } @@ -2125,13 +2150,15 @@ public class FileUtil extends PathUtil { /** * 从文件中读取每一行数据 * - * @param url 文件的URL - * @param charset 字符集 + * @param url 文件的URL + * @param charsetName 字符集 * @return 文件中的每行内容的集合List * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #readLines(URL, Charset)} */ - public static List readLines(URL url, String charset) throws IORuntimeException { - return readLines(url, charset, new ArrayList<>()); + @Deprecated + public static List readLines(URL url, String charsetName) throws IORuntimeException { + return readLines(url, CharsetUtil.charset(charsetName)); } /** @@ -2412,9 +2439,11 @@ public class FileUtil extends PathUtil { * @param isAppend 是否追加 * @return BufferedReader对象 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #getWriter(String, Charset, boolean)} */ + @Deprecated public static BufferedWriter getWriter(String path, String charsetName, boolean isAppend) throws IORuntimeException { - return getWriter(touch(path), Charset.forName(charsetName), isAppend); + return getWriter(path, Charset.forName(charsetName), isAppend); } /** @@ -2438,7 +2467,9 @@ public class FileUtil extends PathUtil { * @param isAppend 是否追加 * @return BufferedReader对象 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #getWriter(File, Charset, boolean)} */ + @Deprecated public static BufferedWriter getWriter(File file, String charsetName, boolean isAppend) throws IORuntimeException { return getWriter(file, Charset.forName(charsetName), isAppend); } diff --git a/hutool-core/src/main/java/cn/hutool/core/io/IoUtil.java b/hutool-core/src/main/java/cn/hutool/core/io/IoUtil.java index f8b2459d5..df058513d 100644 --- a/hutool-core/src/main/java/cn/hutool/core/io/IoUtil.java +++ b/hutool-core/src/main/java/cn/hutool/core/io/IoUtil.java @@ -206,7 +206,9 @@ public class IoUtil extends NioUtil { * @param in 输入流 * @param charsetName 字符集名称 * @return BufferedReader对象 + * @deprecated 请使用 {@link #getReader(InputStream, Charset)} */ + @Deprecated public static BufferedReader getReader(InputStream in, String charsetName) { return getReader(in, Charset.forName(charsetName)); } @@ -290,7 +292,9 @@ public class IoUtil extends NioUtil { * @param out 输入流 * @param charsetName 字符集 * @return OutputStreamWriter对象 + * @deprecated 请使用 {@link #getWriter(OutputStream, Charset)} */ + @Deprecated public static OutputStreamWriter getWriter(OutputStream out, String charsetName) { return getWriter(out, Charset.forName(charsetName)); } @@ -336,7 +340,9 @@ public class IoUtil extends NioUtil { * @param charsetName 字符集 * @return 内容 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #read(InputStream, Charset)} */ + @Deprecated public static String read(InputStream in, String charsetName) throws IORuntimeException { final FastByteArrayOutputStream out = read(in); return StrUtil.isBlank(charsetName) ? out.toString() : out.toString(charsetName); @@ -637,7 +643,9 @@ public class IoUtil extends NioUtil { * @param collection 返回集合 * @return 内容 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #readLines(InputStream, Charset, Collection)} */ + @Deprecated public static > T readLines(InputStream in, String charsetName, T collection) throws IORuntimeException { return readLines(in, CharsetUtil.charset(charsetName), collection); } @@ -697,7 +705,8 @@ public class IoUtil extends NioUtil { /** * 按行读取数据，针对每行的数据做处理
- * {@link Reader}自带编码定义，因此读取数据的编码跟随其编码。 + * {@link Reader}自带编码定义，因此读取数据的编码跟随其编码。
+ * 此方法不会关闭流，除非抛出异常 * * @param reader {@link Reader} * @param lineHandler 行处理接口，实现handle方法用于编辑一行的数据后入到指定地方 @@ -707,15 +716,8 @@ public class IoUtil extends NioUtil { Assert.notNull(reader); Assert.notNull(lineHandler); - // 从返回的内容中读取所需内容 - final BufferedReader bReader = getReader(reader); - String line; - try { - while ((line = bReader.readLine()) != null) { - lineHandler.handle(line); - } - } catch (IOException e) { - throw new IORuntimeException(e); + for (String line : lineIter(reader)) { + lineHandler.handle(line); } } @@ -727,7 +729,9 @@ public class IoUtil extends NioUtil { * @param content 内容 * @param charsetName 编码 * @return 字节流 + * @deprecated 请使用 {@link #toStream(String, Charset)} */ + @Deprecated public static ByteArrayInputStream toStream(String content, String charsetName) { return toStream(content, CharsetUtil.charset(charsetName)); } @@ -1008,7 +1012,9 @@ public class IoUtil extends NioUtil { * @param isCloseOut 写入完毕是否关闭输出流 * @param contents 写入的内容，调用toString()方法，不包括不会自动换行 * @throws IORuntimeException IO异常 + * @deprecated 请使用 {@link #write(OutputStream, Charset, boolean, Object...)} */ + @Deprecated public static void write(OutputStream out, String charsetName, boolean isCloseOut, Object... contents) throws IORuntimeException { write(out, CharsetUtil.charset(charsetName), isCloseOut, contents); } diff --git a/hutool-core/src/test/java/cn/hutool/core/collection/CollStreamUtilTest.java b/hutool-core/src/test/java/cn/hutool/core/collection/CollStreamUtilTest.java index 2d56c0495..c2bad1680 100644 --- a/hutool-core/src/test/java/cn/hutool/core/collection/CollStreamUtilTest.java +++ b/hutool-core/src/test/java/cn/hutool/core/collection/CollStreamUtilTest.java @@ -81,6 +81,7 @@ public class CollStreamUtilTest { Assert.assertEquals(map, compare); } + @Test public void testGroupBy2Key() { Map>> map = CollStreamUtil.groupBy2Key(null, Student::getTermId, Student::getClassId); Assert.assertEquals(map, Collections.EMPTY_MAP); @@ -229,4 +230,4 @@ public class CollStreamUtilTest { private long studentId;//班级id private String name;//学生名称 } -} \ No newline at end of file +} diff --git a/hutool-core/src/test/java/cn/hutool/core/collection/PartitionIterTest.java b/hutool-core/src/test/java/cn/hutool/core/collection/PartitionIterTest.java new file mode 100644 index 000000000..f88ae072f --- /dev/null +++ b/hutool-core/src/test/java/cn/hutool/core/collection/PartitionIterTest.java @@ -0,0 +1,31 @@ +package cn.hutool.core.collection; + +import cn.hutool.core.io.resource.ResourceUtil; +import cn.hutool.core.util.NumberUtil; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class PartitionIterTest { + + @Test + public void iterTest() { + final LineIter lineIter = new LineIter(ResourceUtil.getUtf8Reader("test_lines.csv")); + final PartitionIter iter = new PartitionIter<>(lineIter.iterator(), 3); + for (List lines : iter) { + Assert.assertTrue(lines.size() > 0); + } + } + + @Test + public void iterMaxTest() { + final List list = ListUtil.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 0, 12, 45, 12); + final PartitionIter iter = new PartitionIter<>(list.iterator(), 3); + int max = 0; + for (List lines : iter) { + max = NumberUtil.max(max, NumberUtil.max(lines.toArray(new Integer[0]))); + } + Assert.assertEquals(45, max); + } +} diff --git a/hutool-core/src/test/java/cn/hutool/core/io/IoUtilTest.java b/hutool-core/src/test/java/cn/hutool/core/io/IoUtilTest.java index a6fcf8262..c649d7659 100644 --- a/hutool-core/src/test/java/cn/hutool/core/io/IoUtilTest.java +++ b/hutool-core/src/test/java/cn/hutool/core/io/IoUtilTest.java @@ -4,6 +4,9 @@ import cn.hutool.core.io.resource.ResourceUtil; import org.junit.Assert; import org.junit.Test; +import java.io.BufferedReader; +import java.io.IOException; + public class IoUtilTest { @Test @@ -11,4 +14,13 @@ public class IoUtilTest { final byte[] bytes = IoUtil.readBytes(ResourceUtil.getStream("hutool.jpg")); Assert.assertEquals(22807, bytes.length); } + + @Test + public void readLinesTest(){ + try(BufferedReader reader = ResourceUtil.getUtf8Reader("test_lines.csv");){ + IoUtil.readLines(reader, (LineHandler) Assert::assertNotNull); + } catch (IOException e) { + throw new IORuntimeException(e); + } + } }