奇怪的代码如何隐藏错误？TensorFlow.NET分析

介绍

TensorFlow.NET

研究分析仪报告时引起关注的片段

精致的收藏之旅

private static void _RemoveDefaultAttrs(....)
{
  var producer_op_dict = new Dictionary<string, OpDef>();
  producer_op_list.Op.Select(op =>
  {
    producer_op_dict[op.Name] = op;
    return op;
  }).ToArray();           
  ....
}

var producer_op_dict = new Dictionary<string, OpDef>();

foreach (var op in producer_op_list.Op)
{
  producer_op_dict[op.Name] = op;
}

public GraphDef convert_variables_to_constants(....)
{
  ....
  inference_graph.Node.Select(x => map_name_to_node[x.Name] = x).ToArray();
  ....
}

临时解决方案

public GraphDef convert_variables_to_constants(....)
{
  ....
  var source_op_name = get_input_name(node);
  while(map_name_to_node[source_op_name].Op == "Identity")
  {
    throw new NotImplementedException);
    ....
  }
  ....
}

public static Tensor[] _SoftmaxCrossEntropyWithLogitsGrad(
  Operation op, Tensor[] grads
)
{
  var grad_loss = grads[0];
  var grad_grad = grads[1];
  var softmax_grad = op.outputs[1];
  var grad = _BroadcastMul(grad_loss, softmax_grad);

  var logits = op.inputs[0];
  if(grad_grad != null && !IsZero(grad_grad)) // <=
  {
    throw new NotImplementedException("_SoftmaxCrossEntropyWithLogitsGrad");
  }

  return new Tensor[] 
  {
    grad,
    _BroadcastMul(grad_loss, -nn_ops.log_softmax(logits))
  };
}

private static bool IsZero(Tensor g)
{
  if (new string[] { "ZerosLike", "Zeros" }.Contains(g.op.type))
    return true;

  throw new NotImplementedException("IsZero");
}

张量是张量吗？

private static Tensor[] _ExtractInputShapes(Tensor[] inputs)
{
  var sizes = new Tensor[inputs.Length];
  bool fully_known = true;
  for(int i = 0; i < inputs.Length; i++)
  {
    var x = inputs[i];

    var input_shape = array_ops.shape(x);
    if (!(input_shape is Tensor) || input_shape.op.type != "Const")
    {
      fully_known = false;
      break;
    }

    sizes[i] = input_shape;
  }
  ....
}

彻底检查

public static Tensor[] _BaseFusedBatchNormGrad(....)
{
  ....
  if (data_format == "NCHW") // <=
    throw new NotImplementedException("");

  var results = grad_fun(new FusedBatchNormParams
  {
    YBackprop = grad_y,
    X = x,
    Scale = scale,
    ReserveSpace1 = pop_mean,
    ReserveSpace2 = pop_var,
    ReserveSpace3 = version == 2 ? op.outputs[5] : null,
    Epsilon = epsilon,
    DataFormat = data_format,
    IsTraining = is_training
  });

  var (dx, dscale, doffset) = (results[0], results[1], results[2]);
  if (data_format == "NCHW") // <=
    throw new NotImplementedException("");

  ....
}

• V3021有两个带有相同条件表达式的'if'语句。第一个'if'语句包含方法return。这意味着第二个“ if”语句是毫无意义的nn_grad.cs 230
• V3022表达式'data_format ==“ NCHW”'始终为false。nn_grad.cs 247

选择的幻想

public Tensor Activate(Tensor x, string name = null)
{
  ....
  Tensor negative_part;
  if (Math.Abs(_threshold) > 0.000001f)
  {
    negative_part = gen_ops.relu(-x + _threshold);
  } else
  {
    negative_part = gen_ops.relu(-x + _threshold);
  }
  ....
}

private static Operation _GroupControlDeps(
  string dev, Operation[] deps, string name = null
)
{
  return tf_with(ops.control_dependencies(deps), ctl =>
  {
    if (dev == null)
    {
      return gen_control_flow_ops.no_op(name);
    }
    else
    {
      return gen_control_flow_ops.no_op(name);
    }
  });
}

迟来的支票

public static Tensor[] Input(int[] batch_shape = null,
  TF_DataType dtype = TF_DataType.DtInvalid,
  string name = null,
  bool sparse = false,
  Tensor tensor = null)
{
  var batch_size = batch_shape[0];
  var shape = batch_shape.Skip(1).ToArray(); // <=

  InputLayer input_layer = null;
  if (batch_shape != null)                   // <=
    ....
  else
    ....

  ....
}

另一个“待会儿”？

public MnistDataSet(
  NDArray images, NDArray labels, Type dataType, bool reshape // <=
) 
{
  EpochsCompleted = 0;
  IndexInEpoch = 0;

  NumOfExamples = images.shape[0];

  images = images.reshape(
    images.shape[0], images.shape[1] * images.shape[2]
  );
  images = images.astype(dataType);
  // for debug np.multiply performance
  var sw = new Stopwatch();
  sw.Start();
  images = np.multiply(images, 1.0f / 255.0f);
  sw.Stop();
  Console.WriteLine($"{sw.ElapsedMilliseconds}ms");
  Data = images;

  labels = labels.astype(dataType);
  Labels = labels;
}

难以捉摸的null取消引用

public static Tensor[] _GatherV2Grad(Operation op, Tensor[] grads)
{
  ....
  if((int)axis_static == 0)
  {
    var params_tail_shape = params_shape.slice(new NumSharp.Slice(start:1));
    var values_shape = array_ops.concat(
      new[] { indices_size, params_tail_shape }, 0
    );
    var values = array_ops.reshape(grad, values_shape);
    indices = array_ops.reshape(indices, indices_size);
    return new Tensor[]
    {
      new IndexedSlices(values, indices, params_shape), // <=
      null,
      null
    };
  }
  ....
}

public IndexedSlices(
  Tensor values, Tensor indices, Tensor dense_shape = null
)
{
  _values = values;
  _indices = indices;
  _dense_shape = dense_shape;

  _values.Tag = this; // <=
}

public static Tensor reshape<T1, T2>(T1 tensor, T2 shape, string name = null)
            => gen_array_ops.reshape(tensor, shape, null);

public static Tensor reshape<T1, T2>(T1 tensor, T2 shape, string name = null)
{
  var _op = _op_def_lib._apply_op_helper(
    "Reshape", name, new { tensor, shape }
  );
  return _op.output;
}

public Tensor output => _outputs.FirstOrDefault();

等待不可靠？

private (LoopVar<TItem>, Tensor[]) _BuildLoop<TItem>(
  ....
) where ....
{
  ....
  // Finds the closest enclosing non-None control pivot.
  var outer_context = _outer_context;
  object control_pivot = null;
  while (outer_context != null && control_pivot == null) // <=
  {

  }

  if (control_pivot != null)
  {

  }
  ....
}

打破边界

public TensorShape(int[][] dims)
{
  if(dims.Length == 1)
  {
    switch (dims[0].Length)
    {
      case 0: shape = new Shape(new int[0]); break;
      case 1: shape = Shape.Vector((int)dims[0][0]); break;
      case 2: shape = Shape.Matrix(dims[0][0], dims[1][2]); break; // <=
      default: shape = new Shape(dims[0]); break;
    }
  }
  else
  {
    throw new NotImplementedException("TensorShape int[][] dims");
  }
}

Olepatka？

private void _init_from_args(object initial_value = null, ....) // <=
{
  var init_from_fn = initial_value.GetType().Name == "Func`1"; // <=
  ....
  tf_with(...., scope =>
  {
    ....
    tf_with(...., delegate
    {
      initial_value = ops.convert_to_tensor(  // <=
        init_from_fn ? (initial_value as Func<Tensor>)():initial_value,
        name: "initial_value",
        dtype: dtype
      );
    });
    _shape = shape ?? (initial_value as Tensor).TensorShape;
    _initial_value = initial_value as Tensor; // <=
    ....
    _dtype = _initial_value.dtype.as_base_dtype(); // <=

    if (_in_graph_mode)
    {
      ....

      if (initial_value != null) // <=
      {
        ....
      }

      ....
    }

    ....
  });
}

[DebuggerStepThrough] // with "Just My Code" enabled this lets the 
[DebuggerNonUserCode()]  //debugger break at the origin of the exception
public static void tf_with<T>(
  T py, Action<T> action
) where T : ITensorFlowObject
{
  try
  {
    py.__enter__();
    action(py);
  }
  finally
  {
    py.__exit__();
    py.Dispose();
  }
}

结论