change line sep

This commit is contained in:
Looly
2021-01-20 17:10:45 +08:00
parent 720d24566b
commit 4e38adb32d
1450 changed files with 183940 additions and 183940 deletions

View File

@@ -1,145 +1,145 @@
package cn.hutool.bloomfilter;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.HashUtil;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.BitSet;
/**
* BloomFilter实现方式2此方式使用BitSet存储。<br>
* Hash算法的使用使用固定顺序只需指定个数即可
* @author loolly
*
*/
public class BitSetBloomFilter implements BloomFilter{
private static final long serialVersionUID = 1L;
private final BitSet bitSet;
private final int bitSetSize;
private final int addedElements;
private final int hashFunctionNumber;
/**
* 构造一个布隆过滤器过滤器的容量为c * n 个bit.
*
* @param c 当前过滤器预先开辟的最大包含记录,通常要比预计存入的记录多一倍.
* @param n 当前过滤器预计所要包含的记录.
* @param k 哈希函数的个数等同每条记录要占用的bit数.
*/
public BitSetBloomFilter(int c, int n, int k) {
this.hashFunctionNumber = k;
this.bitSetSize = (int) Math.ceil(c * k);
this.addedElements = n;
this.bitSet = new BitSet(this.bitSetSize);
}
/**
* 通过文件初始化过滤器.
*
* @param path 文件路径
* @param charset 字符集
* @throws IOException IO异常
*/
public void init(String path, String charset) throws IOException {
BufferedReader reader = FileUtil.getReader(path, charset);
try {
String line;
while(true) {
line = reader.readLine();
if(line == null) {
break;
}
this.add(line);
}
}finally {
IoUtil.close(reader);
}
}
@Override
public boolean add(String str) {
if (contains(str)) {
return false;
}
int[] positions = createHashes(str, hashFunctionNumber);
for (int value : positions) {
int position = Math.abs(value % bitSetSize);
bitSet.set(position, true);
}
return true;
}
/**
* 判定是否包含指定字符串
* @param str 字符串
* @return 是否包含,存在误差
*/
@Override
public boolean contains(String str) {
int[] positions = createHashes(str, hashFunctionNumber);
for (int i : positions) {
int position = Math.abs(i % bitSetSize);
if (!bitSet.get(position)) {
return false;
}
}
return true;
}
/**
* @return 得到当前过滤器的错误率.
*/
public double getFalsePositiveProbability() {
// (1 - e^(-k * n / m)) ^ k
return Math.pow((1 - Math.exp(-hashFunctionNumber * (double) addedElements / bitSetSize)), hashFunctionNumber);
}
/**
* 将字符串的字节表示进行多哈希编码.
*
* @param str 待添加进过滤器的字符串字节表示.
* @param hashNumber 要经过的哈希个数.
* @return 各个哈希的结果数组.
*/
public static int[] createHashes(String str, int hashNumber) {
int[] result = new int[hashNumber];
for(int i = 0; i < hashNumber; i++) {
result[i] = hash(str, i);
}
return result;
}
/**
* 计算Hash值
* @param str 被计算Hash的字符串
* @param k Hash算法序号
* @return Hash值
*/
public static int hash(String str, int k) {
switch (k) {
case 0:
return HashUtil.rsHash(str);
case 1:
return HashUtil.jsHash(str);
case 2:
return HashUtil.elfHash(str);
case 3:
return HashUtil.bkdrHash(str);
case 4:
return HashUtil.apHash(str);
case 5:
return HashUtil.djbHash(str);
case 6:
return HashUtil.sdbmHash(str);
case 7:
return HashUtil.pjwHash(str);
default:
return 0;
}
}
package cn.hutool.bloomfilter;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.HashUtil;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.BitSet;
/**
* BloomFilter实现方式2此方式使用BitSet存储。<br>
* Hash算法的使用使用固定顺序只需指定个数即可
* @author loolly
*
*/
public class BitSetBloomFilter implements BloomFilter{
private static final long serialVersionUID = 1L;
private final BitSet bitSet;
private final int bitSetSize;
private final int addedElements;
private final int hashFunctionNumber;
/**
* 构造一个布隆过滤器过滤器的容量为c * n 个bit.
*
* @param c 当前过滤器预先开辟的最大包含记录,通常要比预计存入的记录多一倍.
* @param n 当前过滤器预计所要包含的记录.
* @param k 哈希函数的个数等同每条记录要占用的bit数.
*/
public BitSetBloomFilter(int c, int n, int k) {
this.hashFunctionNumber = k;
this.bitSetSize = (int) Math.ceil(c * k);
this.addedElements = n;
this.bitSet = new BitSet(this.bitSetSize);
}
/**
* 通过文件初始化过滤器.
*
* @param path 文件路径
* @param charset 字符集
* @throws IOException IO异常
*/
public void init(String path, String charset) throws IOException {
BufferedReader reader = FileUtil.getReader(path, charset);
try {
String line;
while(true) {
line = reader.readLine();
if(line == null) {
break;
}
this.add(line);
}
}finally {
IoUtil.close(reader);
}
}
@Override
public boolean add(String str) {
if (contains(str)) {
return false;
}
int[] positions = createHashes(str, hashFunctionNumber);
for (int value : positions) {
int position = Math.abs(value % bitSetSize);
bitSet.set(position, true);
}
return true;
}
/**
* 判定是否包含指定字符串
* @param str 字符串
* @return 是否包含,存在误差
*/
@Override
public boolean contains(String str) {
int[] positions = createHashes(str, hashFunctionNumber);
for (int i : positions) {
int position = Math.abs(i % bitSetSize);
if (!bitSet.get(position)) {
return false;
}
}
return true;
}
/**
* @return 得到当前过滤器的错误率.
*/
public double getFalsePositiveProbability() {
// (1 - e^(-k * n / m)) ^ k
return Math.pow((1 - Math.exp(-hashFunctionNumber * (double) addedElements / bitSetSize)), hashFunctionNumber);
}
/**
* 将字符串的字节表示进行多哈希编码.
*
* @param str 待添加进过滤器的字符串字节表示.
* @param hashNumber 要经过的哈希个数.
* @return 各个哈希的结果数组.
*/
public static int[] createHashes(String str, int hashNumber) {
int[] result = new int[hashNumber];
for(int i = 0; i < hashNumber; i++) {
result[i] = hash(str, i);
}
return result;
}
/**
* 计算Hash值
* @param str 被计算Hash的字符串
* @param k Hash算法序号
* @return Hash值
*/
public static int hash(String str, int k) {
switch (k) {
case 0:
return HashUtil.rsHash(str);
case 1:
return HashUtil.jsHash(str);
case 2:
return HashUtil.elfHash(str);
case 3:
return HashUtil.bkdrHash(str);
case 4:
return HashUtil.apHash(str);
case 5:
return HashUtil.djbHash(str);
case 6:
return HashUtil.sdbmHash(str);
case 7:
return HashUtil.pjwHash(str);
default:
return 0;
}
}
}

View File

@@ -1,32 +1,32 @@
package cn.hutool.bloomfilter;
/**
* 布隆过滤器工具
*
* @author looly
* @since 4.1.5
*/
public class BloomFilterUtil {
/**
* 创建一个BitSet实现的布隆过滤器过滤器的容量为c * n 个bit.
*
* @param c 当前过滤器预先开辟的最大包含记录,通常要比预计存入的记录多一倍.
* @param n 当前过滤器预计所要包含的记录.
* @param k 哈希函数的个数等同每条记录要占用的bit数.
* @return BitSetBloomFilter
*/
public static BitSetBloomFilter createBitSet(int c, int n, int k) {
return new BitSetBloomFilter(c, n, k);
}
/**
* 创建BitMap实现的布隆过滤器
*
* @param m BitMap的大小
* @return BitMapBloomFilter
*/
public static BitMapBloomFilter createBitMap(int m) {
return new BitMapBloomFilter(m);
}
}
package cn.hutool.bloomfilter;
/**
* 布隆过滤器工具
*
* @author looly
* @since 4.1.5
*/
public class BloomFilterUtil {
/**
* 创建一个BitSet实现的布隆过滤器过滤器的容量为c * n 个bit.
*
* @param c 当前过滤器预先开辟的最大包含记录,通常要比预计存入的记录多一倍.
* @param n 当前过滤器预计所要包含的记录.
* @param k 哈希函数的个数等同每条记录要占用的bit数.
* @return BitSetBloomFilter
*/
public static BitSetBloomFilter createBitSet(int c, int n, int k) {
return new BitSetBloomFilter(c, n, k);
}
/**
* 创建BitMap实现的布隆过滤器
*
* @param m BitMap的大小
* @return BitMapBloomFilter
*/
public static BitMapBloomFilter createBitMap(int m) {
return new BitMapBloomFilter(m);
}
}

View File

@@ -1,7 +1,7 @@
/**
* BitMap实现
*
* @author looly
*
*/
/**
* BitMap实现
*
* @author looly
*
*/
package cn.hutool.bloomfilter.bitMap;

View File

@@ -1,25 +1,25 @@
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
/**
* 默认Bloom过滤器使用Java自带的Hash算法
*
* @author loolly
*/
public class DefaultFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public DefaultFilter(long maxValue, int machineNumber) {
super(maxValue, machineNumber);
}
public DefaultFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.javaDefaultHash(str) % size;
}
}
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
/**
* 默认Bloom过滤器使用Java自带的Hash算法
*
* @author loolly
*/
public class DefaultFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public DefaultFilter(long maxValue, int machineNumber) {
super(maxValue, machineNumber);
}
public DefaultFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.javaDefaultHash(str) % size;
}
}

View File

@@ -1,21 +1,21 @@
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class ELFFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public ELFFilter(long maxValue, int machineNumber) {
super(maxValue, machineNumber);
}
public ELFFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.elfHash(str) % size;
}
}
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class ELFFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public ELFFilter(long maxValue, int machineNumber) {
super(maxValue, machineNumber);
}
public ELFFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.elfHash(str) % size;
}
}

View File

@@ -1,21 +1,21 @@
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class FNVFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public FNVFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public FNVFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.fnvHash(str);
}
}
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class FNVFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public FNVFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public FNVFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.fnvHash(str);
}
}

View File

@@ -1,31 +1,31 @@
package cn.hutool.bloomfilter.filter;
public class HfFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public HfFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public HfFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
int length = str.length() ;
long hash = 0;
for (int i = 0; i < length; i++) {
hash += str.charAt(i) * 3 * i;
}
if (hash < 0) {
hash = -hash;
}
return hash % size;
}
}
package cn.hutool.bloomfilter.filter;
public class HfFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public HfFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public HfFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
int length = str.length() ;
long hash = 0;
for (int i = 0; i < length; i++) {
hash += str.charAt(i) * 3 * i;
}
if (hash < 0) {
hash = -hash;
}
return hash % size;
}
}

View File

@@ -1,24 +1,24 @@
package cn.hutool.bloomfilter.filter;
public class HfIpFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public HfIpFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public HfIpFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
int length = str.length();
long hash = 0;
for (int i = 0; i < length; i++) {
hash += str.charAt(i % 4) ^ str.charAt(i);
}
return hash % size;
}
}
package cn.hutool.bloomfilter.filter;
public class HfIpFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public HfIpFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public HfIpFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
int length = str.length();
long hash = 0;
for (int i = 0; i < length; i++) {
hash += str.charAt(i % 4) ^ str.charAt(i);
}
return hash % size;
}
}

View File

@@ -1,30 +1,30 @@
package cn.hutool.bloomfilter.filter;
public class JSFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public JSFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public JSFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
int hash = 1315423911;
for (int i = 0; i < str.length(); i++) {
hash ^= ((hash << 5) + str.charAt(i) + (hash >> 2));
}
if(hash<0) {
hash*=-1 ;
}
return hash % size;
}
}
package cn.hutool.bloomfilter.filter;
public class JSFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public JSFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public JSFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
int hash = 1315423911;
for (int i = 0; i < str.length(); i++) {
hash ^= ((hash << 5) + str.charAt(i) + (hash >> 2));
}
if(hash<0) {
hash*=-1 ;
}
return hash % size;
}
}

View File

@@ -1,21 +1,21 @@
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class PJWFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public PJWFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public PJWFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.pjwHash(str) % size;
}
}
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class PJWFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public PJWFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public PJWFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.pjwHash(str) % size;
}
}

View File

@@ -1,21 +1,21 @@
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class RSFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public RSFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public RSFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.rsHash(str) % size;
}
}
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class RSFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public RSFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public RSFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.rsHash(str) % size;
}
}

View File

@@ -1,21 +1,21 @@
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class SDBMFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public SDBMFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public SDBMFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.sdbmHash(str) % size;
}
}
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class SDBMFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public SDBMFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public SDBMFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.sdbmHash(str) % size;
}
}

View File

@@ -1,22 +1,22 @@
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class TianlFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public TianlFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public TianlFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.tianlHash(str) % size;
}
}
package cn.hutool.bloomfilter.filter;
import cn.hutool.core.util.HashUtil;
public class TianlFilter extends AbstractFilter {
private static final long serialVersionUID = 1L;
public TianlFilter(long maxValue, int machineNum) {
super(maxValue, machineNum);
}
public TianlFilter(long maxValue) {
super(maxValue);
}
@Override
public long hash(String str) {
return HashUtil.tianlHash(str) % size;
}
}

View File

@@ -1,7 +1,7 @@
/**
* 各种Hash算法的过滤器实现
*
* @author looly
*
*/
/**
* 各种Hash算法的过滤器实现
*
* @author looly
*
*/
package cn.hutool.bloomfilter.filter;

View File

@@ -1,7 +1,7 @@
/**
* 布隆过滤提供一些Hash算法的布隆过滤
*
* @author looly
*
*/
/**
* 布隆过滤提供一些Hash算法的布隆过滤
*
* @author looly
*
*/
package cn.hutool.bloomfilter;

View File

@@ -1,55 +1,55 @@
package cn.hutool.bloomfilter;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import cn.hutool.bloomfilter.bitMap.IntMap;
import cn.hutool.bloomfilter.bitMap.LongMap;
public class BitMapBloomFilterTest {
@Test
public void filterTest() {
BitMapBloomFilter filter = new BitMapBloomFilter(10);
filter.add("123");
filter.add("abc");
filter.add("ddd");
Assert.assertTrue(filter.contains("abc"));
Assert.assertTrue(filter.contains("ddd"));
Assert.assertTrue(filter.contains("123"));
}
@Test
@Ignore
public void testIntMap(){
IntMap intMap = new IntMap();
for (int i = 0 ; i < 32; i++) {
intMap.add(i);
}
intMap.remove(30);
for (int i = 0; i < 32; i++) {
System.out.println(i + "是否存在-->" + intMap.contains(i));
}
}
@Test
@Ignore
public void testLongMap(){
LongMap longMap = new LongMap();
for (int i = 0 ; i < 64; i++) {
longMap.add(i);
}
longMap.remove(30);
for (int i = 0; i < 64; i++) {
System.out.println(i + "是否存在-->" + longMap.contains(i));
}
}
}
package cn.hutool.bloomfilter;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import cn.hutool.bloomfilter.bitMap.IntMap;
import cn.hutool.bloomfilter.bitMap.LongMap;
public class BitMapBloomFilterTest {
@Test
public void filterTest() {
BitMapBloomFilter filter = new BitMapBloomFilter(10);
filter.add("123");
filter.add("abc");
filter.add("ddd");
Assert.assertTrue(filter.contains("abc"));
Assert.assertTrue(filter.contains("ddd"));
Assert.assertTrue(filter.contains("123"));
}
@Test
@Ignore
public void testIntMap(){
IntMap intMap = new IntMap();
for (int i = 0 ; i < 32; i++) {
intMap.add(i);
}
intMap.remove(30);
for (int i = 0; i < 32; i++) {
System.out.println(i + "是否存在-->" + intMap.contains(i));
}
}
@Test
@Ignore
public void testLongMap(){
LongMap longMap = new LongMap();
for (int i = 0 ; i < 64; i++) {
longMap.add(i);
}
longMap.remove(30);
for (int i = 0; i < 64; i++) {
System.out.println(i + "是否存在-->" + longMap.contains(i));
}
}
}