Unfortunately, right now that is how formulas work in both designers. Any formula that references a field in a table/collection regardless of row recalculates when the table state changes. This is something we are looking into, but there are cases where you DO want this behavior so its not as simple as just fixing it. 

For now you can use JS to handle the formulas and these will bypass row add/delete 

const fields = {
  sourceCalcFieldCol: { fieldId: 7 },
  destCalcFieldCol: { fieldId: 8 }
};
// Do some calculation
const performCalculation = (sourceVal) => {
  return sourceVal * 2;
};
// Handle calculation change events
const handleCalculationChange = async (index) => {
  if (index) {
    const sourceValSingle = LFForm.getFieldValues({
      ...fields.sourceCalcFieldCol,
      index
    });
    return LFForm.setFieldValues(
      { ...fields.destCalcFieldCol, index },
      performCalculation(sourceValSingle)
    );
  }
  const sourceValList = LFForm.getFieldValues(
    fields.sourceCalcFieldCol
  );
  const sourceValAsList = Array.isArray(sourceValList) ? sourceValList : [sourceValList];
  const destVal = sourceValAsList.map((val) => performCalculation(val));
  return LFForm.setFieldValues(fields.destCalcFieldCol, destVal);
};
const main = async () => {
  // register change events
  LFForm.onFieldChange(async (e) => {
    const fieldsChanged = e.options;
    for (const row of fieldsChanged) {
      const index = row.index;
      await handleCalculationChange(index);
    }
  }, fields.sourceCalcFieldCol);
  // initial calculations
  await handleCalculationChange();
};
main().catch(console.warn);