"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java" between
solr-8.4.0-src.tgz and solr-8.4.1-src.tgz

About: Solr is the search platform from the Apache Lucene project. Its major features include full-text search, hit highlighting, faceted search, caching, replication, and a web admin interface.

PackageStoreAPI.java  (solr-8.4.0-src.tgz):PackageStoreAPI.java  (solr-8.4.1-src.tgz)
skipping to change at line 38 skipping to change at line 38
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Supplier; import java.util.function.Supplier;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.solr.api.Command; import org.apache.solr.api.Command;
import org.apache.solr.api.EndPoint; import org.apache.solr.api.EndPoint;
import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.common.MapWriter; import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.Utils;
import org.apache.solr.core.BlobRepository; import org.apache.solr.core.BlobRepository;
import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreContainer;
skipping to change at line 68 skipping to change at line 67
import org.apache.zookeeper.server.ByteBufferInputStream; import org.apache.zookeeper.server.ByteBufferInputStream;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
public class PackageStoreAPI { public class PackageStoreAPI {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup ().lookupClass()); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup ().lookupClass());
public static final String PACKAGESTORE_DIRECTORY = "filestore"; public static final String PACKAGESTORE_DIRECTORY = "filestore";
public static final String TRUSTED_DIR = "_trusted_";
public static final String KEYS_DIR = "/_trusted_/keys";
private final CoreContainer coreContainer; private final CoreContainer coreContainer;
PackageStore packageStore; PackageStore packageStore;
public final FSRead readAPI = new FSRead(); public final FSRead readAPI = new FSRead();
public final FSWrite writeAPI = new FSWrite(); public final FSWrite writeAPI = new FSWrite();
public PackageStoreAPI(CoreContainer coreContainer) { public PackageStoreAPI(CoreContainer coreContainer) {
this.coreContainer = coreContainer; this.coreContainer = coreContainer;
packageStore = new DistribPackageStore(coreContainer); packageStore = new DistribPackageStore(coreContainer);
} }
skipping to change at line 100 skipping to change at line 101
l.remove(coreContainer.getZkController().getNodeName()); l.remove(coreContainer.getZkController().getNodeName());
Collections.shuffle(l, BlobRepository.RANDOM); Collections.shuffle(l, BlobRepository.RANDOM);
return l; return l;
} }
public void validateFiles(List<String> files, boolean validateSignatures, Cons umer<String> errs) { public void validateFiles(List<String> files, boolean validateSignatures, Cons umer<String> errs) {
for (String path : files) { for (String path : files) {
try { try {
PackageStore.FileType type = packageStore.getType(path, true); PackageStore.FileType type = packageStore.getType(path, true);
if (type != PackageStore.FileType.FILE) { if (type != PackageStore.FileType.FILE) {
errs.accept("No such file : " + path); errs.accept("No such file: " + path);
continue; continue;
} }
packageStore.get(path, entry -> { packageStore.get(path, entry -> {
if (entry.getMetaData().signatures == null || if (entry.getMetaData().signatures == null ||
entry.getMetaData().signatures.isEmpty()) { entry.getMetaData().signatures.isEmpty()) {
errs.accept(path + " has no signature"); errs.accept(path + " has no signature");
return; return;
} }
if (validateSignatures) { if (validateSignatures) {
try { try {
validate(entry.meta.signatures, entry); packageStore.refresh(KEYS_DIR);
} catch (SolrException e) { validate(entry.meta.signatures, entry, false);
log.error("error validating package artifact", e); } catch (Exception e) {
log.error("Error validating package artifact", e);
errs.accept(e.getMessage()); errs.accept(e.getMessage());
} }
} }
}, false); }, false);
} catch (Exception e) { } catch (Exception e) {
log.error("Error reading file ", e); log.error("Error reading file ", e);
errs.accept("Error reading file " + path + " " + e.getMessage()); errs.accept("Error reading file " + path + " " + e.getMessage());
} }
} }
skipping to change at line 137 skipping to change at line 139
@EndPoint( @EndPoint(
path = "/cluster/files/*", path = "/cluster/files/*",
method = SolrRequest.METHOD.PUT, method = SolrRequest.METHOD.PUT,
permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM) permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
public class FSWrite { public class FSWrite {
static final String TMP_ZK_NODE = "/packageStoreWriteInProgress"; static final String TMP_ZK_NODE = "/packageStoreWriteInProgress";
@Command @Command
public void upload(SolrQueryRequest req, SolrQueryResponse rsp) { public void upload(SolrQueryRequest req, SolrQueryResponse rsp) {
if(!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {
throw new RuntimeException(PackageAPI.ERR_MSG); throw new RuntimeException(PackageAPI.ERR_MSG);
} }
try { try {
coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE, "true" .getBytes(UTF_8), coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE, "true" .getBytes(UTF_8),
CreateMode.EPHEMERAL, true); CreateMode.EPHEMERAL, true);
Iterable<ContentStream> streams = req.getContentStreams(); Iterable<ContentStream> streams = req.getContentStreams();
if (streams == null) throw new SolrException(SolrException.ErrorCode.BAD _REQUEST, "no payload"); if (streams == null) throw new SolrException(SolrException.ErrorCode.BAD _REQUEST, "no payload");
String path = req.getPathTemplateValues().get("*"); String path = req.getPathTemplateValues().get("*");
if (path == null) { if (path == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path" ); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path" );
} }
validateName(path); validateName(path, true);
ContentStream stream = streams.iterator().next(); ContentStream stream = streams.iterator().next();
try { try {
ByteBuffer buf = SimplePostTool.inputStreamToByteArray(stream.getStrea m()); ByteBuffer buf = SimplePostTool.inputStreamToByteArray(stream.getStrea m());
String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(buf));
List<String> signatures = readSignatures(req, buf); List<String> signatures = readSignatures(req, buf);
Map<String, Object> vals = new HashMap<>(); MetaData meta = _createJsonMetaData(buf, signatures);
vals.put(MetaData.SHA512, sha512);
if (signatures != null) {
vals.put("sig", signatures);
}
PackageStore.FileType type = packageStore.getType(path, true); PackageStore.FileType type = packageStore.getType(path, true);
if(type != PackageStore.FileType.NOFILE) { if(type != PackageStore.FileType.NOFILE) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path already exists "+ path); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path already exists "+ path);
} }
packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals), p ath)); packageStore.put(new PackageStore.FileEntry(buf, meta, path));
rsp.add(CommonParams.FILE, path); rsp.add(CommonParams.FILE, path);
} catch (IOException e) { } catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
} }
} catch (InterruptedException e) { } catch (InterruptedException e) {
log.error("Unexpected error", e); log.error("Unexpected error", e);
} catch (KeeperException.NodeExistsException e) { } catch (KeeperException.NodeExistsException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A write i s already in process , try later"); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A write i s already in process , try later");
} catch (KeeperException e) { } catch (KeeperException e) {
log.error("Unexpected error", e); log.error("Unexpected error", e);
skipping to change at line 190 skipping to change at line 187
log.error("Unexpected error ", e); log.error("Unexpected error ", e);
} }
} }
} }
private List<String> readSignatures(SolrQueryRequest req, ByteBuffer buf) private List<String> readSignatures(SolrQueryRequest req, ByteBuffer buf)
throws SolrException, IOException { throws SolrException, IOException {
String[] signatures = req.getParams().getParams("sig"); String[] signatures = req.getParams().getParams("sig");
if (signatures == null || signatures.length == 0) return null; if (signatures == null || signatures.length == 0) return null;
List<String> sigs = Arrays.asList(signatures); List<String> sigs = Arrays.asList(signatures);
packageStore.refresh(KEYS_DIR);
validate(sigs, buf); validate(sigs, buf);
return sigs; return sigs;
} }
public void validate(List<String> sigs, private void validate(List<String> sigs,
ByteBuffer buf) throws SolrException, IOException { ByteBuffer buf) throws SolrException, IOException {
Map<String, byte[]> keys = CloudUtil.getTrustedKeys( Map<String, byte[]> keys = packageStore.getKeys();
coreContainer.getZkController().getZkClient(), "exe");
if (keys == null || keys.isEmpty()) { if (keys == null || keys.isEmpty()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"ZK does not have any keys"); "package store does not have any keys");
} }
CryptoKeys cryptoKeys = null; CryptoKeys cryptoKeys = null;
try { try {
cryptoKeys = new CryptoKeys(keys); cryptoKeys = new CryptoKeys(keys);
} catch (Exception e) { } catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error parsing public keys in ZooKeeper"); "Error parsing public keys in Package store");
} }
for (String sig : sigs) { for (String sig : sigs) {
if (cryptoKeys.verify(sig, buf) == null) { if (cryptoKeys.verify(sig, buf) == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signatur e does not match any public key : " + sig +" len: "+buf.limit()+ " content sha5 12: "+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signatur e does not match any public key : " + sig +" len: "+buf.limit()+ " content sha5 12: "+
DigestUtils.sha512Hex(new ByteBufferInputStream(buf))); DigestUtils.sha512Hex(new ByteBufferInputStream(buf)));
} }
} }
} }
} }
/**
* Creates a JSON string with the metadata
* @lucene.internal
*/
public static MetaData _createJsonMetaData(ByteBuffer buf, List<String> signat
ures) throws IOException {
String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(buf));
Map<String, Object> vals = new HashMap<>();
vals.put(MetaData.SHA512, sha512);
if (signatures != null) {
vals.put("sig", signatures);
}
return new MetaData(vals);
}
@EndPoint( @EndPoint(
path = "/node/files/*", path = "/node/files/*",
method = SolrRequest.METHOD.GET, method = SolrRequest.METHOD.GET,
permission = PermissionNameProvider.Name.FILESTORE_READ_PERM) permission = PermissionNameProvider.Name.FILESTORE_READ_PERM)
public class FSRead { public class FSRead {
@Command @Command
public void read(SolrQueryRequest req, SolrQueryResponse rsp) { public void read(SolrQueryRequest req, SolrQueryResponse rsp) {
String path = req.getPathTemplateValues().get("*"); String path = req.getPathTemplateValues().get("*");
String pathCopy = path; String pathCopy = path;
if (req.getParams().getBool("sync", false)) {
try {
packageStore.syncToAllNodes(path);
return;
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error ge
tting file ", e);
}
}
String getFrom = req.getParams().get("getFrom"); String getFrom = req.getParams().get("getFrom");
if (getFrom != null) { if (getFrom != null) {
coreContainer.getUpdateShardHandler().getUpdateExecutor().submit(() -> { coreContainer.getUpdateShardHandler().getUpdateExecutor().submit(() -> {
log.debug("Downloading file {}", pathCopy); log.debug("Downloading file {}", pathCopy);
try { try {
packageStore.fetch(pathCopy, getFrom); packageStore.fetch(pathCopy, getFrom);
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to download file: " + pathCopy, e); log.error("Failed to download file: " + pathCopy, e);
} }
log.info("downloaded file: {}", pathCopy); log.info("downloaded file: {}", pathCopy);
skipping to change at line 288 skipping to change at line 307
} catch (IOException e) { } catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading file" + path); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading file" + path);
} }
}, false); }, false);
}); });
} }
} }
static class MetaData implements MapWriter { public static class MetaData implements MapWriter {
public static final String SHA512 = "sha512"; public static final String SHA512 = "sha512";
String sha512; String sha512;
List<String> signatures; List<String> signatures;
Map<String, Object> otherAttribs; Map<String, Object> otherAttribs;
public MetaData(Map m) { public MetaData(Map m) {
m = Utils.getDeepCopy(m, 3); m = Utils.getDeepCopy(m, 3);
this.sha512 = (String) m.remove(SHA512); this.sha512 = (String) m.remove(SHA512);
this.signatures = (List<String>) m.remove("sig"); this.signatures = (List<String>) m.remove("sig");
this.otherAttribs = m; this.otherAttribs = m;
skipping to change at line 313 skipping to change at line 332
ew.putIfNotNull("sha512", sha512); ew.putIfNotNull("sha512", sha512);
ew.putIfNotNull("sig", signatures); ew.putIfNotNull("sig", signatures);
if (!otherAttribs.isEmpty()) { if (!otherAttribs.isEmpty()) {
otherAttribs.forEach(ew.getBiConsumer()); otherAttribs.forEach(ew.getBiConsumer());
} }
} }
} }
static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$><?{}[]|:;!"; static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$><?{}[]|:;!";
public static void validateName(String path) { public static void validateName(String path, boolean failForTrusted) {
if (path == null) { if (path == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty path") ; throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty path") ;
} }
List<String> parts = StrUtils.splitSmart(path, '/', true); List<String> parts = StrUtils.splitSmart(path, '/', true);
for (String part : parts) { for (String part : parts) {
if (part.charAt(0) == '.') { if (part.charAt(0) == '.') {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "cannot sta rt with period"); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "cannot sta rt with period");
} }
for (int i = 0; i < part.length(); i++) { for (int i = 0; i < part.length(); i++) {
for (int j = 0; j < INVALIDCHARS.length(); j++) { for (int j = 0; j < INVALIDCHARS.length(); j++) {
if (part.charAt(i) == INVALIDCHARS.charAt(j)) if (part.charAt(i) == INVALIDCHARS.charAt(j))
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unsupp orted char in file name: " + part); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unsupp orted char in file name: " + part);
} }
} }
} }
if (failForTrusted && TRUSTED_DIR.equals(parts.get(0))) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "trying to wr
ite into /_trusted_/ directory");
}
} }
/**Validate a file for signature
*
* @param sigs the signatures. atleast one should succeed
* @param entry The file details
* @param isFirstAttempt If there is a failure
*/
public void validate(List<String> sigs, public void validate(List<String> sigs,
PackageStore.FileEntry entry) throws SolrException { PackageStore.FileEntry entry,
Map<String, byte[]> keys = CloudUtil.getTrustedKeys( boolean isFirstAttempt) throws SolrException, IOException
coreContainer.getZkController().getZkClient(), "exe"); {
if (!isFirstAttempt) {
//we are retrying because last validation failed.
// get all keys again and try again
packageStore.refresh(KEYS_DIR);
}
Map<String, byte[]> keys = packageStore.getKeys();
if (keys == null || keys.isEmpty()) { if (keys == null || keys.isEmpty()) {
if(isFirstAttempt) {
validate(sigs, entry, false);
return;
}
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"ZooKeeper does not have any public keys"); "Packagestore does not have any public keys");
} }
CryptoKeys cryptoKeys = null; CryptoKeys cryptoKeys = null;
try { try {
cryptoKeys = new CryptoKeys(keys); cryptoKeys = new CryptoKeys(keys);
} catch (Exception e) { } catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error parsing public keys in ZooKeeper"); "Error parsing public keys in ZooKeeper");
} }
for (String sig : sigs) { for (String sig : sigs) {
Supplier<String> errMsg = () -> "Signature does not match any public key : " + sig + "sha256 "+ entry.getMetaData().sha512; Supplier<String> errMsg = () -> "Signature does not match any public key : " + sig + "sha256 " + entry.getMetaData().sha512;
if (entry.getBuffer() != null) { if (entry.getBuffer() != null) {
if (cryptoKeys.verify(sig, entry.getBuffer()) == null) { if (cryptoKeys.verify(sig, entry.getBuffer()) == null) {
if(isFirstAttempt) {
validate(sigs, entry, false);
return;
}
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.ge t()); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.ge t());
} }
} else { } else {
InputStream inputStream = entry.getInputStream(); InputStream inputStream = entry.getInputStream();
if (cryptoKeys.verify(sig, inputStream) == null) { if (cryptoKeys.verify(sig, inputStream) == null) {
if(isFirstAttempt) {
validate(sigs, entry, false);
return;
}
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.ge t()); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.ge t());
} }
} }
} }
} }
} }
 End of changes. 25 change blocks. 
27 lines changed or deleted 77 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)