Yes you can use a script based validator to do that, but the implementation can be a bit complicated. First use the workItem.getValue() function to retrieve the current values of the attribute. This will be an array of JavaScript objects, with each containing four attributes - _eQualifiedClassName, iconUrl, id and label (you will be interested in the last two only). Loop through the array and process the labels - you can put them into a "Set" for easy access. Evaluate the chosen values, and return Status.OK_STATUS or a new error depending on the condition. The main body of the function will look like this.
var enumList = workItem.getValue("enumList");
var enumSet = new Set();
for (var i=0; i<enumList.length; i++) {
enumSet.add(enumList[i].label);
}
console.log("enumSet: " + enumSet);
if (enumSet.has("A") && enumSet.size>1) {
return new Status(Severity["ERROR"], "When A is chosen, no other values are allowed");
} else if (enumSet.has("B") && enumSet.has("C") && enumSet.has("D") && enumSet.has("E")) {
return new Status(Severity["ERROR"], "B, C, D&E are not allowed as a combination");
} else {
return Status.OK_STATUS;
}